freemo
af9db5ea28
CI / lint (pull_request) Successful in 32s
CI / quality (pull_request) Successful in 32s
CI / typecheck (pull_request) Successful in 46s
CI / build (pull_request) Successful in 31s
CI / helm (pull_request) Successful in 35s
CI / security (pull_request) Successful in 1m19s
CI / unit_tests (pull_request) Failing after 6m45s
CI / docker (pull_request) Has been skipped
CI / e2e_tests (pull_request) Successful in 15m36s
CI / coverage (pull_request) Successful in 10m43s
CI / integration_tests (pull_request) Failing after 23m29s
CI / status-check (pull_request) Failing after 3s
CI / benchmark-publish (pull_request) Has been skipped
CI / benchmark-regression (pull_request) Successful in 56m58s
1238 lines
41 KiB
Python
1238 lines
41 KiB
Python
"""Behave step implementations for context strategy registry feature."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import MappingProxyType
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from pydantic import ValidationError
|
|
|
|
from cleveragents.application.services.strategy_registry import (
|
|
StrategyNotFoundError,
|
|
StrategyRegistrationError,
|
|
StrategyRegistry,
|
|
)
|
|
from cleveragents.domain.models.acms.crp import (
|
|
ContextFragment,
|
|
ContextRequest,
|
|
FragmentProvenance,
|
|
)
|
|
from cleveragents.domain.models.acms.strategy import (
|
|
BackendSet,
|
|
ContextStrategy,
|
|
ContextStrategyResult,
|
|
PlanContext,
|
|
StrategyCapabilities,
|
|
StrategyConfig,
|
|
)
|
|
from cleveragents.domain.models.acms.strategy_stubs import (
|
|
BUILTIN_STRATEGY_CLASSES,
|
|
DEFAULT_ENABLED_STRATEGIES,
|
|
)
|
|
from cleveragents.domain.models.acms.stubs import (
|
|
InMemoryGraphBackend,
|
|
InMemoryTextBackend,
|
|
InMemoryVectorBackend,
|
|
)
|
|
from cleveragents.domain.models.acms.temporal_stubs import InMemoryTemporalBackend
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _all_builtins() -> dict[str, ContextStrategy]:
|
|
"""Instantiate all built-in strategies keyed by name."""
|
|
instances: dict[str, ContextStrategy] = {}
|
|
for cls in BUILTIN_STRATEGY_CLASSES:
|
|
inst = cls()
|
|
instances[inst.name] = inst
|
|
return instances
|
|
|
|
|
|
def _register_all_builtins(
|
|
registry: StrategyRegistry,
|
|
*,
|
|
use_default_enabled: bool = False,
|
|
) -> None:
|
|
"""Register all built-in strategies in the registry."""
|
|
for cls in BUILTIN_STRATEGY_CLASSES:
|
|
inst = cls()
|
|
enabled = (
|
|
inst.name in DEFAULT_ENABLED_STRATEGIES if use_default_enabled else True
|
|
)
|
|
registry.register(
|
|
inst,
|
|
config=StrategyConfig(enabled=enabled),
|
|
is_builtin=True,
|
|
)
|
|
if use_default_enabled:
|
|
registry.set_enabled(list(DEFAULT_ENABLED_STRATEGIES))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("all built-in strategies are instantiated")
|
|
def step_given_all_builtins(context: Context) -> None:
|
|
context.strategies = _all_builtins()
|
|
|
|
|
|
@given("an empty strategy registry")
|
|
def step_given_empty_registry(context: Context) -> None:
|
|
context.registry = StrategyRegistry()
|
|
|
|
|
|
@given('I register the "{name}" built-in strategy')
|
|
def step_given_register_builtin(context: Context, name: str) -> None:
|
|
strategies = _all_builtins()
|
|
context.registry.register(strategies[name], is_builtin=True)
|
|
|
|
|
|
@given("a registry with all built-in strategies")
|
|
def step_given_registry_all(context: Context) -> None:
|
|
context.registry = StrategyRegistry()
|
|
_register_all_builtins(context.registry)
|
|
|
|
|
|
@given("a registry with all built-in strategies using default enabled list")
|
|
def step_given_registry_default_enabled(context: Context) -> None:
|
|
context.registry = StrategyRegistry()
|
|
_register_all_builtins(context.registry, use_default_enabled=True)
|
|
|
|
|
|
@given("a BackendSet with text backend only")
|
|
def step_given_backends_text(context: Context) -> None:
|
|
context.backend_set = BackendSet(text=InMemoryTextBackend())
|
|
|
|
|
|
@given("a BackendSet with vector backend only")
|
|
def step_given_backends_vector(context: Context) -> None:
|
|
context.backend_set = BackendSet(vector=InMemoryVectorBackend())
|
|
|
|
|
|
@given("a BackendSet with graph backend only")
|
|
def step_given_backends_graph(context: Context) -> None:
|
|
context.backend_set = BackendSet(graph=InMemoryGraphBackend())
|
|
|
|
|
|
@given("a BackendSet with all backends")
|
|
def step_given_backends_all(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
text=InMemoryTextBackend(),
|
|
vector=InMemoryVectorBackend(),
|
|
graph=InMemoryGraphBackend(),
|
|
)
|
|
|
|
|
|
@given("a BackendSet with no backends")
|
|
def step_given_backends_none(context: Context) -> None:
|
|
context.backend_set = BackendSet()
|
|
|
|
|
|
@given("a default ContextRequest")
|
|
def step_given_default_request(context: Context) -> None:
|
|
context.request = ContextRequest(query="test query")
|
|
|
|
|
|
@given("a default PlanContext")
|
|
def step_given_default_plan_context(context: Context) -> None:
|
|
context.plan_context = PlanContext()
|
|
|
|
|
|
@given("a ContextStrategyResult with unordered fragments")
|
|
def step_given_unordered_result(context: Context) -> None:
|
|
prov = FragmentProvenance(resource_uri="res://test")
|
|
context.result = ContextStrategyResult(
|
|
strategy_name="test",
|
|
fragments=(
|
|
ContextFragment(
|
|
uko_node="uko:Bravo",
|
|
content="b",
|
|
detail_depth=1,
|
|
token_count=10,
|
|
relevance_score=0.5,
|
|
provenance=prov,
|
|
),
|
|
ContextFragment(
|
|
uko_node="uko:Alpha",
|
|
content="a",
|
|
detail_depth=1,
|
|
token_count=10,
|
|
relevance_score=0.9,
|
|
provenance=prov,
|
|
),
|
|
ContextFragment(
|
|
uko_node="uko:Alpha",
|
|
content="a2",
|
|
detail_depth=1,
|
|
token_count=10,
|
|
relevance_score=0.5,
|
|
provenance=prov,
|
|
),
|
|
),
|
|
total_fragments=3,
|
|
tokens_used=30,
|
|
)
|
|
|
|
|
|
@given("a ContextStrategyResult with no fragments")
|
|
def step_given_empty_result(context: Context) -> None:
|
|
context.result = ContextStrategyResult(strategy_name="empty")
|
|
|
|
|
|
@given("a registry with a stale enabled list entry")
|
|
def step_given_stale_enabled(context: Context) -> None:
|
|
"""Create a registry where the enabled list references a name not registered."""
|
|
context.registry = StrategyRegistry()
|
|
_register_all_builtins(context.registry)
|
|
# Use the public test helper to inject a stale entry
|
|
context.registry.inject_stale_enabled_entry("ghost-strategy")
|
|
|
|
|
|
@given("a registry with a strategy that declares no resource types")
|
|
def step_given_no_resource_types(context: Context) -> None:
|
|
context.registry = StrategyRegistry()
|
|
|
|
class _NoResourceTypes:
|
|
@property
|
|
def name(self) -> str:
|
|
return "no-resource-types"
|
|
|
|
@property
|
|
def capabilities(self) -> StrategyCapabilities:
|
|
return StrategyCapabilities(
|
|
uses_text=True,
|
|
resource_types=(),
|
|
quality_score=0.3,
|
|
)
|
|
|
|
def can_handle(self, request: ContextRequest, backends: BackendSet) -> float:
|
|
return 0.3
|
|
|
|
def assemble(
|
|
self,
|
|
request: ContextRequest,
|
|
backends: BackendSet,
|
|
budget: int,
|
|
plan_context: PlanContext,
|
|
) -> list[ContextFragment]:
|
|
return []
|
|
|
|
def explain(self) -> str:
|
|
return "test"
|
|
|
|
context.registry.register(_NoResourceTypes())
|
|
|
|
|
|
@given("a registry with a strategy that declares no backend capabilities")
|
|
def step_given_no_capabilities(context: Context) -> None:
|
|
context.registry = StrategyRegistry()
|
|
|
|
class _NoBackend:
|
|
@property
|
|
def name(self) -> str:
|
|
return "no-backends"
|
|
|
|
@property
|
|
def capabilities(self) -> StrategyCapabilities:
|
|
return StrategyCapabilities(
|
|
uses_text=False,
|
|
uses_vector=False,
|
|
uses_graph=False,
|
|
uses_temporal=False,
|
|
resource_types=("*",),
|
|
quality_score=0.1,
|
|
)
|
|
|
|
def can_handle(self, request: ContextRequest, backends: BackendSet) -> float:
|
|
return 0.1
|
|
|
|
def assemble(
|
|
self,
|
|
request: ContextRequest,
|
|
backends: BackendSet,
|
|
budget: int,
|
|
plan_context: PlanContext,
|
|
) -> list[ContextFragment]:
|
|
return []
|
|
|
|
def explain(self) -> str:
|
|
return "test"
|
|
|
|
context.registry.register(_NoBackend())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I register the "{name}" built-in strategy')
|
|
def step_when_register_builtin(context: Context, name: str) -> None:
|
|
strategies = _all_builtins()
|
|
context.registry.register(strategies[name], is_builtin=True)
|
|
|
|
|
|
@when("I register all 6 built-in strategies")
|
|
def step_when_register_all(context: Context) -> None:
|
|
_register_all_builtins(context.registry)
|
|
|
|
|
|
@when('I attempt to register "{name}" again')
|
|
def step_when_register_duplicate(context: Context, name: str) -> None:
|
|
strategies = _all_builtins()
|
|
try:
|
|
context.registry.register(strategies[name], is_builtin=True)
|
|
context.error = None
|
|
except StrategyRegistrationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I attempt to register a strategy with an empty name")
|
|
def step_when_register_empty_name(context: Context) -> None:
|
|
class _Empty:
|
|
@property
|
|
def name(self) -> str:
|
|
return ""
|
|
|
|
@property
|
|
def capabilities(self) -> StrategyCapabilities:
|
|
return StrategyCapabilities(uses_text=True, resource_types=("*",))
|
|
|
|
def can_handle(self, request: ContextRequest, backends: BackendSet) -> float:
|
|
return 0.0
|
|
|
|
def assemble(
|
|
self,
|
|
request: ContextRequest,
|
|
backends: BackendSet,
|
|
budget: int,
|
|
plan_context: PlanContext,
|
|
) -> list[ContextFragment]:
|
|
return []
|
|
|
|
def explain(self) -> str:
|
|
return ""
|
|
|
|
try:
|
|
context.registry.register(_Empty())
|
|
context.error = None
|
|
except StrategyRegistrationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I get the strategy "{name}"')
|
|
def step_when_get_strategy(context: Context, name: str) -> None:
|
|
context.fetched_strategy = context.registry.get(name)
|
|
|
|
|
|
@when('I attempt to get the strategy "{name}"')
|
|
def step_when_get_nonexistent(context: Context, name: str) -> None:
|
|
try:
|
|
context.registry.get(name)
|
|
context.error = None
|
|
except StrategyNotFoundError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I set the enabled list to "{names}"')
|
|
def step_when_set_enabled(context: Context, names: str) -> None:
|
|
name_list = [n.strip().strip('"') for n in names.split(",")]
|
|
context.registry.set_enabled(name_list)
|
|
|
|
|
|
@when('I attempt to set the enabled list to "{names}"')
|
|
def step_when_set_enabled_bad(context: Context, names: str) -> None:
|
|
name_list = [n.strip().strip('"') for n in names.split(",")]
|
|
try:
|
|
context.registry.set_enabled(name_list)
|
|
context.error = None
|
|
except StrategyNotFoundError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I disable the strategy "{name}"')
|
|
def step_when_disable(context: Context, name: str) -> None:
|
|
context.registry.update_config(name, enabled=False)
|
|
|
|
|
|
@when('I get the config for "{name}"')
|
|
def step_when_get_config(context: Context, name: str) -> None:
|
|
context.strategy_config = context.registry.get_config(name)
|
|
|
|
|
|
@when('I update config for "{name}" with timeout_seconds {timeout:d}')
|
|
def step_when_update_config(context: Context, name: str, timeout: int) -> None:
|
|
context.registry.update_config(name, timeout_seconds=timeout)
|
|
|
|
|
|
@when("each strategy assembles with budget {budget:d}")
|
|
def step_when_assemble_all(context: Context, budget: int) -> None:
|
|
context.assemble_results = {}
|
|
for name, strategy in context.strategies.items():
|
|
result = strategy.assemble(
|
|
context.request,
|
|
context.backend_set,
|
|
budget,
|
|
context.plan_context,
|
|
)
|
|
context.assemble_results[name] = result
|
|
|
|
|
|
@when('I register a strategy from module "{module_path}"')
|
|
def step_when_register_from_module(context: Context, module_path: str) -> None:
|
|
# Use "simple-keyword" as the name since the scenario asserts on it.
|
|
context.registry.register_from_module(
|
|
name="simple-keyword",
|
|
module_path=module_path,
|
|
)
|
|
|
|
|
|
@when('I attempt to register a strategy from module "{module_path}"')
|
|
def step_when_register_from_module_fail(context: Context, module_path: str) -> None:
|
|
try:
|
|
context.registry.register_from_module(name="test", module_path=module_path)
|
|
context.error = None
|
|
except StrategyRegistrationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I get the entry for "{name}"')
|
|
def step_when_get_entry(context: Context, name: str) -> None:
|
|
context.fetched_entry = context.registry.get_entry(name)
|
|
|
|
|
|
@when('I attempt to get the entry for "{name}"')
|
|
def step_when_get_entry_fail(context: Context, name: str) -> None:
|
|
try:
|
|
context.registry.get_entry(name)
|
|
context.error = None
|
|
except StrategyNotFoundError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I attempt to unregister "{name}"')
|
|
def step_when_unregister_fail(context: Context, name: str) -> None:
|
|
try:
|
|
context.registry.unregister(name)
|
|
context.error = None
|
|
except StrategyNotFoundError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I unregister "{name}"')
|
|
def step_when_unregister(context: Context, name: str) -> None:
|
|
context.registry.unregister(name)
|
|
|
|
|
|
@when("I clear the registry")
|
|
def step_when_clear(context: Context) -> None:
|
|
context.registry.clear()
|
|
|
|
|
|
@when("I validate the registry")
|
|
def step_when_validate(context: Context) -> None:
|
|
context.validation_warnings = context.registry.validate_registry()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("every strategy should satisfy the ContextStrategy protocol")
|
|
def step_then_protocol(context: Context) -> None:
|
|
for name, strategy in context.strategies.items():
|
|
assert isinstance(strategy, ContextStrategy), (
|
|
f"Strategy '{name}' does not satisfy ContextStrategy protocol"
|
|
)
|
|
|
|
|
|
@then("every strategy should have a non-empty name")
|
|
def step_then_nonempty_name(context: Context) -> None:
|
|
for name, strategy in context.strategies.items():
|
|
assert strategy.name, f"Strategy has empty name (key={name})"
|
|
|
|
|
|
@then("every strategy should have a capabilities object")
|
|
def step_then_has_capabilities(context: Context) -> None:
|
|
for name, strategy in context.strategies.items():
|
|
caps = strategy.capabilities
|
|
assert isinstance(caps, StrategyCapabilities), (
|
|
f"Strategy '{name}' capabilities is not StrategyCapabilities"
|
|
)
|
|
|
|
|
|
@then('the strategy "{name}" should have quality score {score:g}')
|
|
def step_then_quality_score(context: Context, name: str, score: float) -> None:
|
|
strategy = context.strategies[name]
|
|
assert strategy.capabilities.quality_score == score, (
|
|
f"Expected {score}, got {strategy.capabilities.quality_score}"
|
|
)
|
|
|
|
|
|
@then('the strategy registry should contain "{name}"')
|
|
def step_then_contains(context: Context, name: str) -> None:
|
|
assert context.registry.is_registered(name), (
|
|
f"'{name}' not in registry: {context.registry.list_all()}"
|
|
)
|
|
|
|
|
|
@then('the strategy registry should not contain "{name}"')
|
|
def step_then_not_contains(context: Context, name: str) -> None:
|
|
assert not context.registry.is_registered(name), (
|
|
f"'{name}' unexpectedly in registry"
|
|
)
|
|
|
|
|
|
@then("the registry size should be {size:d}")
|
|
def step_then_size(context: Context, size: int) -> None:
|
|
assert len(context.registry) == size, (
|
|
f"Expected {size}, got {len(context.registry)}"
|
|
)
|
|
|
|
|
|
@then("the registry should contain {count:d} strategies")
|
|
def step_then_count(context: Context, count: int) -> None:
|
|
assert len(context.registry) == count, (
|
|
f"Expected {count}, got {len(context.registry)}"
|
|
)
|
|
|
|
|
|
@then("a StrategyRegistrationError should be raised")
|
|
def step_then_reg_error(context: Context) -> None:
|
|
assert isinstance(context.error, StrategyRegistrationError), (
|
|
f"Expected StrategyRegistrationError, got {type(context.error)}"
|
|
)
|
|
assert str(context.error), "Error message should not be empty"
|
|
|
|
|
|
@then("a StrategyNotFoundError should be raised")
|
|
def step_then_not_found_error(context: Context) -> None:
|
|
assert isinstance(context.error, StrategyNotFoundError), (
|
|
f"Expected StrategyNotFoundError, got {type(context.error)}"
|
|
)
|
|
assert str(context.error), "Error message should not be empty"
|
|
|
|
|
|
@then('the strategy name should be "{name}"')
|
|
def step_then_strategy_name(context: Context, name: str) -> None:
|
|
assert context.fetched_strategy.name == name
|
|
|
|
|
|
@then("the registry should list {count:d} strategies")
|
|
def step_then_list_count(context: Context, count: int) -> None:
|
|
all_names = context.registry.list_all()
|
|
assert len(all_names) == count, (
|
|
f"Expected {count}, got {len(all_names)}: {all_names}"
|
|
)
|
|
|
|
|
|
@then("the list should be sorted alphabetically")
|
|
def step_then_sorted(context: Context) -> None:
|
|
names = context.registry.list_all()
|
|
assert names == sorted(names), f"Names not sorted: {names}"
|
|
|
|
|
|
@then('the enabled strategies should be "{names}"')
|
|
def step_then_enabled(context: Context, names: str) -> None:
|
|
expected = [n.strip().strip('"') for n in names.split(",")]
|
|
actual = context.registry.list_enabled()
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then('the enabled strategies should not include "{name}"')
|
|
def step_then_not_enabled(context: Context, name: str) -> None:
|
|
enabled = context.registry.list_enabled()
|
|
assert name not in enabled, f"'{name}' unexpectedly in enabled list: {enabled}"
|
|
|
|
|
|
@then("the timeout_seconds should be {val:d}")
|
|
def step_then_timeout(context: Context, val: int) -> None:
|
|
assert context.strategy_config.timeout_seconds == val
|
|
|
|
|
|
@then("the max_fragments should be {val:d}")
|
|
def step_then_max_fragments(context: Context, val: int) -> None:
|
|
assert context.strategy_config.max_fragments == val
|
|
|
|
|
|
@then("the circuit_breaker_threshold should be {val:d}")
|
|
def step_then_cb(context: Context, val: int) -> None:
|
|
assert context.strategy_config.circuit_breaker_threshold == val
|
|
|
|
|
|
@then('the config for "{name}" should have timeout_seconds {val:d}')
|
|
def step_then_config_timeout(context: Context, name: str, val: int) -> None:
|
|
config = context.registry.get_config(name)
|
|
assert config.timeout_seconds == val, (
|
|
f"Expected {val}, got {config.timeout_seconds}"
|
|
)
|
|
|
|
|
|
@then('"{name}" can_handle should return {score:g}')
|
|
def step_then_can_handle(context: Context, name: str, score: float) -> None:
|
|
strategies = _all_builtins()
|
|
strategy = strategies[name]
|
|
result = strategy.can_handle(context.request, context.backend_set)
|
|
assert abs(result - score) < 1e-6, f"Expected {score}, got {result} for '{name}'"
|
|
|
|
|
|
@then("every strategy should return an empty fragment list")
|
|
def step_then_empty_assemble(context: Context) -> None:
|
|
for name, result in context.assemble_results.items():
|
|
assert result == [], f"Strategy '{name}' returned non-empty: {result}"
|
|
|
|
|
|
@then("the fragments should be sorted by relevance_score DESC then uko_node ASC")
|
|
def step_then_sorted_fragments(context: Context) -> None:
|
|
frags = context.result.fragments
|
|
assert len(frags) == 3
|
|
# relevance DESC: 0.9 first, then 0.5s; within 0.5: Alpha < Bravo
|
|
assert frags[0].relevance_score == 0.9
|
|
assert frags[0].uko_node == "uko:Alpha"
|
|
assert frags[1].relevance_score == 0.5
|
|
assert frags[1].uko_node == "uko:Alpha"
|
|
assert frags[2].relevance_score == 0.5
|
|
assert frags[2].uko_node == "uko:Bravo"
|
|
|
|
|
|
@then("the total_fragments should be {val:d}")
|
|
def step_then_total_fragments(context: Context, val: int) -> None:
|
|
assert context.result.total_fragments == val
|
|
|
|
|
|
@then("the tokens_used should be {val:d}")
|
|
def step_then_tokens_used(context: Context, val: int) -> None:
|
|
assert context.result.tokens_used == val
|
|
|
|
|
|
@then('validation warnings should mention "{text}"')
|
|
def step_then_validation_mentions(context: Context, text: str) -> None:
|
|
combined = " ".join(context.validation_warnings)
|
|
assert text in combined, (
|
|
f"Expected '{text}' in warnings: {context.validation_warnings}"
|
|
)
|
|
|
|
|
|
@then('the entry name should be "{name}"')
|
|
def step_then_entry_name(context: Context, name: str) -> None:
|
|
assert context.fetched_entry.name == name
|
|
|
|
|
|
@then("the entry should be marked as builtin")
|
|
def step_then_entry_builtin(context: Context) -> None:
|
|
assert context.fetched_entry.is_builtin, "Expected entry to be builtin"
|
|
|
|
|
|
@then("the builtin list should contain {count:d} strategies")
|
|
def step_then_builtin_count(context: Context, count: int) -> None:
|
|
builtins = context.registry.list_builtin()
|
|
assert len(builtins) == count, (
|
|
f"Expected {count} builtins, got {len(builtins)}: {builtins}"
|
|
)
|
|
|
|
|
|
@then('"{name}" should be in the registry via contains')
|
|
def step_then_contains_in(context: Context, name: str) -> None:
|
|
assert name in context.registry, f"'{name}' not in registry via __contains__"
|
|
|
|
|
|
@then('"{name}" should not be in the registry via contains')
|
|
def step_then_not_contains_in(context: Context, name: str) -> None:
|
|
assert name not in context.registry, (
|
|
f"'{name}' unexpectedly in registry via __contains__"
|
|
)
|
|
|
|
|
|
@then("there should be no validation warnings")
|
|
def step_then_no_warnings(context: Context) -> None:
|
|
assert context.validation_warnings == [], (
|
|
f"Expected no warnings, got: {context.validation_warnings}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Boundary test steps (TEST-1)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I attempt to create a StrategyConfig with {field} set to {value:d}")
|
|
def step_when_invalid_config(context: Context, field: str, value: int) -> None:
|
|
try:
|
|
StrategyConfig.model_validate({field: value})
|
|
context.error = None
|
|
except ValidationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I attempt to create a StrategyCapabilities with quality_score {value:g}")
|
|
def step_when_invalid_quality(context: Context, value: float) -> None:
|
|
try:
|
|
StrategyCapabilities(uses_text=True, quality_score=value)
|
|
context.error = None
|
|
except ValidationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I create a StrategyCapabilities with quality_score {value:g}")
|
|
def step_when_valid_quality(context: Context, value: float) -> None:
|
|
context.created_capabilities = StrategyCapabilities(
|
|
uses_text=True,
|
|
quality_score=value,
|
|
)
|
|
|
|
|
|
@then("a Pydantic ValidationError should be raised")
|
|
def step_then_pydantic_validation_error(context: Context) -> None:
|
|
assert context.error is not None, (
|
|
"Expected Pydantic ValidationError but none raised"
|
|
)
|
|
assert isinstance(context.error, ValidationError), (
|
|
f"Expected pydantic.ValidationError, got {type(context.error).__name__}"
|
|
)
|
|
|
|
|
|
@then("the quality_score should be {value:g}")
|
|
def step_then_quality_value(context: Context, value: float) -> None:
|
|
assert abs(context.created_capabilities.quality_score - value) < 1e-9, (
|
|
f"Expected {value}, got {context.created_capabilities.quality_score}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bug regression steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when(
|
|
'I update config for "{name}" with max_workers {mw:d}'
|
|
" and circuit_breaker_threshold {cb:d}"
|
|
)
|
|
def step_when_update_mw_cb(context: Context, name: str, mw: int, cb: int) -> None:
|
|
context.registry.update_config(name, max_workers=mw, circuit_breaker_threshold=cb)
|
|
|
|
|
|
@then('the config for "{name}" should have max_workers {val:d}')
|
|
def step_then_config_max_workers(context: Context, name: str, val: int) -> None:
|
|
config = context.registry.get_config(name)
|
|
assert config.max_workers == val, f"Expected {val}, got {config.max_workers}"
|
|
|
|
|
|
@then('the config for "{name}" should have circuit_breaker_threshold {val:d}')
|
|
def step_then_config_cb(context: Context, name: str, val: int) -> None:
|
|
config = context.registry.get_config(name)
|
|
assert config.circuit_breaker_threshold == val, (
|
|
f"Expected {val}, got {config.circuit_breaker_threshold}"
|
|
)
|
|
|
|
|
|
@when("I attempt to register a non-protocol object")
|
|
def step_when_register_non_protocol(context: Context) -> None:
|
|
try:
|
|
# Pass name explicitly so register() reaches the isinstance check
|
|
context.registry.register("not-a-strategy", name="fake") # type: ignore[arg-type]
|
|
context.error = None
|
|
except StrategyRegistrationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I attempt to register a non-protocol object without a name")
|
|
def step_when_register_non_protocol_no_name(context: Context) -> None:
|
|
try:
|
|
# No name= override — verifies guard ordering: isinstance check
|
|
# must happen before strategy.name access (bug 1.1 fix).
|
|
context.registry.register(object()) # type: ignore[arg-type]
|
|
context.error = None
|
|
except StrategyRegistrationError as exc:
|
|
context.error = exc
|
|
except AttributeError as exc: # pragma: no cover
|
|
# This would indicate the bug is still present
|
|
context.error = exc
|
|
|
|
|
|
@then("every strategy should return a non-empty explain string")
|
|
def step_then_explain_nonempty(context: Context) -> None:
|
|
for name, strategy in context.strategies.items():
|
|
explanation = strategy.explain()
|
|
assert isinstance(explanation, str), (
|
|
f"Strategy '{name}' explain() returned {type(explanation)}, expected str"
|
|
)
|
|
assert explanation.strip(), f"Strategy '{name}' returned empty explain string"
|
|
|
|
|
|
@when('I register the "{name}" built-in strategy with enabled False')
|
|
def step_when_register_builtin_disabled(context: Context, name: str) -> None:
|
|
strategies = _all_builtins()
|
|
context.registry.register(
|
|
strategies[name],
|
|
config=StrategyConfig(enabled=False),
|
|
is_builtin=True,
|
|
)
|
|
|
|
|
|
@when('I update config for "{name}" with enabled True')
|
|
def step_when_update_enabled_true(context: Context, name: str) -> None:
|
|
context.registry.update_config(name, enabled=True)
|
|
|
|
|
|
@when('I update config for "{name}" with enabled False')
|
|
def step_when_update_enabled_false(context: Context, name: str) -> None:
|
|
context.registry.update_config(name, enabled=False)
|
|
|
|
|
|
@then('the enabled strategies should be ""')
|
|
def step_then_enabled_empty(context: Context) -> None:
|
|
actual = context.registry.list_enabled()
|
|
assert actual == [], f"Expected empty enabled list, got {actual}"
|
|
|
|
|
|
@then('the enabled strategies should include "{name}"')
|
|
def step_then_enabled_includes(context: Context, name: str) -> None:
|
|
enabled = context.registry.list_enabled()
|
|
assert name in enabled, f"'{name}' not in enabled list: {enabled}"
|
|
|
|
|
|
@when('I register a strategy named "{name}" from module "{module_path}"')
|
|
def step_when_register_from_module_with_name(
|
|
context: Context, name: str, module_path: str
|
|
) -> None:
|
|
context.registry.register_from_module(name=name, module_path=module_path)
|
|
|
|
|
|
@when('I attempt to update config for "{name}" with timeout_seconds {value:d}')
|
|
def step_when_update_config_bad_timeout(
|
|
context: Context, name: str, value: int
|
|
) -> None:
|
|
try:
|
|
context.registry.update_config(name, timeout_seconds=value)
|
|
context.error = None
|
|
except ValidationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when('I attempt to update config for "{name}" with max_fragments {value:d}')
|
|
def step_when_update_config_bad_fragments(
|
|
context: Context, name: str, value: int
|
|
) -> None:
|
|
try:
|
|
context.registry.update_config(name, max_fragments=value)
|
|
context.error = None
|
|
except ValidationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Thread safety steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("10 threads concurrently register and query the registry")
|
|
def step_when_concurrent_register_query(context: Context) -> None:
|
|
"""Exercise the registry from multiple threads simultaneously."""
|
|
import threading as _threading
|
|
|
|
errors: list[Exception] = []
|
|
barrier = _threading.Barrier(10)
|
|
|
|
def _worker(idx: int) -> None:
|
|
try:
|
|
barrier.wait(timeout=5)
|
|
# Mix of reads and writes
|
|
if idx % 3 == 0:
|
|
# Write: register a unique strategy
|
|
name = f"thread-strategy-{idx}"
|
|
|
|
class _Stub:
|
|
@property
|
|
def name(self) -> str:
|
|
return name
|
|
|
|
@property
|
|
def capabilities(self) -> StrategyCapabilities:
|
|
return StrategyCapabilities(
|
|
uses_text=True, resource_types=("*",), quality_score=0.1
|
|
)
|
|
|
|
def can_handle(
|
|
self, request: ContextRequest, backends: BackendSet
|
|
) -> float:
|
|
return 0.1
|
|
|
|
def assemble(
|
|
self,
|
|
request: ContextRequest,
|
|
backends: BackendSet,
|
|
budget: int,
|
|
plan_context: PlanContext,
|
|
) -> list[ContextFragment]:
|
|
return []
|
|
|
|
def explain(self) -> str:
|
|
return f"thread stub {idx}"
|
|
|
|
context.registry.register(_Stub())
|
|
elif idx % 3 == 1:
|
|
# Read: list enabled
|
|
context.registry.list_enabled()
|
|
else:
|
|
# Read: list all
|
|
context.registry.list_all()
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
|
|
threads = [_threading.Thread(target=_worker, args=(i,)) for i in range(10)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join(timeout=10)
|
|
|
|
context.thread_errors = errors
|
|
|
|
|
|
@then("no thread should have raised an exception")
|
|
def step_then_no_thread_errors(context: Context) -> None:
|
|
assert not context.thread_errors, f"Thread errors: {context.thread_errors}"
|
|
|
|
|
|
@then("the registry should contain at least {count:d} strategies")
|
|
def step_then_at_least_count(context: Context, count: int) -> None:
|
|
actual = len(context.registry)
|
|
assert actual >= count, f"Expected at least {count}, got {actual}"
|
|
|
|
|
|
@then("every thread-registered strategy should be present")
|
|
def step_then_thread_strategies_present(context: Context) -> None:
|
|
for idx in [0, 3, 6, 9]:
|
|
name = f"thread-strategy-{idx}"
|
|
assert context.registry.is_registered(name), (
|
|
f"Thread-registered '{name}' missing from registry"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temporal/cold-tier backend steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# update_config: resource_types and extra steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I update config for "{name}" with resource_types "{rt1}", "{rt2}"')
|
|
def step_when_update_resource_types(
|
|
context: Context, name: str, rt1: str, rt2: str
|
|
) -> None:
|
|
context.registry.update_config(name, resource_types=(rt1, rt2))
|
|
|
|
|
|
@then('the config for "{name}" should have resource_types "{rt1}", "{rt2}"')
|
|
def step_then_resource_types(context: Context, name: str, rt1: str, rt2: str) -> None:
|
|
cfg = context.registry.get_config(name)
|
|
assert cfg.resource_types == (rt1, rt2), (
|
|
f"Expected ({rt1!r}, {rt2!r}), got {cfg.resource_types}"
|
|
)
|
|
|
|
|
|
@when('I update config for "{name}" with extra key "{key}" value "{val}"')
|
|
def step_when_update_extra(context: Context, name: str, key: str, val: str) -> None:
|
|
context.registry.update_config(name, extra={key: val})
|
|
|
|
|
|
@then('the config for "{name}" extra should contain key "{key}"')
|
|
def step_then_extra_contains(context: Context, name: str, key: str) -> None:
|
|
cfg = context.registry.get_config(name)
|
|
assert key in cfg.extra, f"Key '{key}' not in extra: {cfg.extra}"
|
|
|
|
|
|
@then('the config for "{name}" extra should be immutable')
|
|
def step_then_extra_immutable(context: Context, name: str) -> None:
|
|
cfg = context.registry.get_config(name)
|
|
assert isinstance(cfg.extra, MappingProxyType), (
|
|
f"Expected MappingProxyType, got {type(cfg.extra).__name__}"
|
|
)
|
|
try:
|
|
cfg.extra["hack"] = "should fail" # type: ignore[index]
|
|
raise AssertionError("MappingProxyType should reject item assignment")
|
|
except TypeError:
|
|
pass # Expected — MappingProxyType is immutable
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MappingProxyType coercion validator steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I create a StrategyConfig with extra as a plain dict")
|
|
def step_when_config_plain_dict_extra(context: Context) -> None:
|
|
context.created_config = StrategyConfig(extra={"key": "val"}) # type: ignore[arg-type]
|
|
|
|
|
|
@then("the extra field should be a MappingProxyType")
|
|
def step_then_extra_is_mapping_proxy(context: Context) -> None:
|
|
assert isinstance(context.created_config.extra, MappingProxyType), (
|
|
f"Expected MappingProxyType, got {type(context.created_config.extra).__name__}"
|
|
)
|
|
|
|
|
|
@when("I attempt to create a StrategyConfig with extra as an integer")
|
|
def step_when_config_int_extra(context: Context) -> None:
|
|
try:
|
|
StrategyConfig(extra=42) # type: ignore[arg-type]
|
|
context.error = None
|
|
except ValidationError as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I create a ContextStrategyResult with stats as a plain dict")
|
|
def step_when_result_plain_dict_stats(context: Context) -> None:
|
|
context.created_result = ContextStrategyResult(
|
|
strategy_name="test",
|
|
stats={"ops": 10}, # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
@then("the stats field should be a MappingProxyType")
|
|
def step_then_stats_is_mapping_proxy(context: Context) -> None:
|
|
assert isinstance(context.created_result.stats, MappingProxyType), (
|
|
f"Expected MappingProxyType, got {type(context.created_result.stats).__name__}"
|
|
)
|
|
|
|
|
|
@given("a BackendSet with graph and temporal backends")
|
|
def step_given_backends_graph_temporal(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
graph=InMemoryGraphBackend(),
|
|
temporal=InMemoryTemporalBackend(),
|
|
)
|
|
|
|
|
|
@given("a BackendSet with temporal backend only")
|
|
def step_given_backends_temporal_only(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
temporal=InMemoryTemporalBackend(),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Populated backend helpers for real retrieval tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_populated_text_backend() -> PopulatedTextBackend:
|
|
"""Create a text backend pre-loaded with test data."""
|
|
return PopulatedTextBackend()
|
|
|
|
|
|
def _make_populated_vector_backend() -> PopulatedVectorBackend:
|
|
"""Create a vector backend pre-loaded with test data."""
|
|
return PopulatedVectorBackend()
|
|
|
|
|
|
def _make_populated_graph_backend() -> PopulatedGraphBackend:
|
|
"""Create a graph backend pre-loaded with test data."""
|
|
return PopulatedGraphBackend()
|
|
|
|
|
|
def _make_populated_temporal_backend() -> InMemoryTemporalBackend:
|
|
"""Create a temporal backend pre-loaded with test data."""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from cleveragents.domain.models.acms.temporal import TemporalMetadata, TemporalNode
|
|
|
|
backend = InMemoryTemporalBackend()
|
|
now = datetime.now(tz=UTC)
|
|
node = TemporalNode(
|
|
node_uri="uko:plan/test-plan_v1",
|
|
source_resource="res://test-resource",
|
|
source_path="src/test.py",
|
|
temporal=TemporalMetadata(
|
|
valid_from=now - timedelta(hours=1),
|
|
is_current=True,
|
|
),
|
|
)
|
|
backend.store_node(node)
|
|
return backend
|
|
|
|
|
|
class PopulatedTextBackend:
|
|
"""Text backend that returns pre-loaded test results."""
|
|
|
|
def search(
|
|
self,
|
|
query: str,
|
|
*,
|
|
scope: frozenset[str],
|
|
max_results: int = 20,
|
|
) -> list:
|
|
from cleveragents.domain.models.acms.backends import TextResult
|
|
|
|
if not query:
|
|
raise ValueError("query must be non-empty")
|
|
if max_results < 1:
|
|
raise ValueError("max_results must be positive")
|
|
return [
|
|
TextResult(
|
|
uko_uri="uko:class/AuthManager",
|
|
content=f"Authentication manager class handling {query}",
|
|
score=0.9,
|
|
),
|
|
TextResult(
|
|
uko_uri="uko:function/authenticate",
|
|
content=f"Function that authenticates users for {query}",
|
|
score=0.7,
|
|
),
|
|
][:max_results]
|
|
|
|
|
|
class PopulatedVectorBackend:
|
|
"""Vector backend that returns pre-loaded test results."""
|
|
|
|
def similarity_search(
|
|
self,
|
|
embedding: list[float],
|
|
*,
|
|
scope: frozenset[str],
|
|
top_k: int = 20,
|
|
) -> list:
|
|
from cleveragents.domain.models.acms.backends import VectorResult
|
|
|
|
if not embedding:
|
|
raise ValueError("embedding must be non-empty")
|
|
if top_k < 1:
|
|
raise ValueError("top_k must be positive")
|
|
return [
|
|
VectorResult(
|
|
uko_uri="uko:class/AuthManager",
|
|
content="Authentication manager with JWT support",
|
|
score=0.85,
|
|
),
|
|
VectorResult(
|
|
uko_uri="uko:class/SessionManager",
|
|
content="Session management for authenticated users",
|
|
score=0.72,
|
|
),
|
|
][:top_k]
|
|
|
|
|
|
class PopulatedGraphBackend:
|
|
"""Graph backend that returns pre-loaded test results."""
|
|
|
|
def sparql_query(
|
|
self,
|
|
query: str,
|
|
*,
|
|
scope: frozenset[str],
|
|
):
|
|
from cleveragents.domain.models.acms.backends import GraphResult
|
|
|
|
if not query:
|
|
raise ValueError("query must be non-empty")
|
|
return GraphResult(
|
|
triples=[
|
|
("uko:class/AuthManager", "rdf:type", "uko:Class"),
|
|
("uko:class/AuthManager", "uko:hasMethod", "uko:method/authenticate"),
|
|
]
|
|
)
|
|
|
|
def get_triples(self, subject: str):
|
|
from cleveragents.domain.models.acms.backends import GraphResult
|
|
|
|
if not subject:
|
|
raise ValueError("subject must be non-empty")
|
|
return GraphResult(
|
|
triples=[
|
|
(subject, "rdf:type", "uko:Class"),
|
|
(subject, "uko:hasMethod", "uko:method/authenticate"),
|
|
]
|
|
)
|
|
|
|
def traverse(self, start: str, *, depth: int = 2):
|
|
from cleveragents.domain.models.acms.backends import GraphResult
|
|
|
|
if not start:
|
|
raise ValueError("start must be non-empty")
|
|
if depth < 0:
|
|
raise ValueError("depth must be non-negative")
|
|
return GraphResult(
|
|
triples=[
|
|
(start, "rdf:type", "uko:Class"),
|
|
(start, "uko:hasMethod", "uko:method/authenticate"),
|
|
(start, "uko:dependsOn", "uko:class/SessionManager"),
|
|
]
|
|
)
|
|
|
|
|
|
@given("a BackendSet with a populated text backend")
|
|
def step_given_populated_text_backend(context: Context) -> None:
|
|
context.backend_set = BackendSet(text=_make_populated_text_backend())
|
|
|
|
|
|
@given("a BackendSet with a populated vector backend")
|
|
def step_given_populated_vector_backend(context: Context) -> None:
|
|
context.backend_set = BackendSet(vector=_make_populated_vector_backend())
|
|
|
|
|
|
@given("a BackendSet with a populated graph backend")
|
|
def step_given_populated_graph_backend(context: Context) -> None:
|
|
context.backend_set = BackendSet(graph=_make_populated_graph_backend())
|
|
|
|
|
|
@given("a BackendSet with all populated backends")
|
|
def step_given_all_populated_backends(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
text=_make_populated_text_backend(),
|
|
vector=_make_populated_vector_backend(),
|
|
graph=_make_populated_graph_backend(),
|
|
)
|
|
|
|
|
|
@given("a BackendSet with populated graph and temporal backends")
|
|
def step_given_populated_graph_temporal_backends(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
graph=_make_populated_graph_backend(),
|
|
temporal=_make_populated_temporal_backend(),
|
|
)
|
|
|
|
|
|
@given("a BackendSet with a populated temporal backend")
|
|
def step_given_populated_temporal_backend(context: Context) -> None:
|
|
context.backend_set = BackendSet(
|
|
temporal=_make_populated_temporal_backend(),
|
|
)
|
|
|
|
|
|
@given('a ContextRequest with query "{query}"')
|
|
def step_given_request_with_query(context: Context, query: str) -> None:
|
|
context.request = ContextRequest(query=query)
|
|
|
|
|
|
@given('a ContextRequest with focus "{focus}"')
|
|
def step_given_request_with_focus(context: Context, focus: str) -> None:
|
|
context.request = ContextRequest(focus=[focus])
|
|
|
|
|
|
@when('the "{name}" strategy assembles with budget {budget:d}')
|
|
def step_when_strategy_assembles(context: Context, name: str, budget: int) -> None:
|
|
strategies = _all_builtins()
|
|
strategy = strategies[name]
|
|
context.single_strategy_result = strategy.assemble(
|
|
context.request,
|
|
context.backend_set,
|
|
budget,
|
|
context.plan_context,
|
|
)
|
|
|
|
|
|
@then('the "{name}" strategy should return at least 1 fragment')
|
|
def step_then_strategy_returns_fragments(context: Context, name: str) -> None:
|
|
result = context.single_strategy_result
|
|
assert len(result) >= 1, (
|
|
f"Strategy '{name}' returned {len(result)} fragments, expected at least 1"
|
|
)
|