Files
temp/features/steps/acms_service_coverage_boost_steps.py
aditya 42a32c2709 fix(acms): align budget allocation formula and protocol signatures with spec
Align ACMS pipeline protocol signatures and allocation formula with the
specification (docs/specification.md §42630-42937):

- BudgetAllocator.allocate() now accepts optional request: ContextRequest
  parameter per BudgetAllocatorProtocol spec (§44754-44766), enabling
  future request-aware allocation strategies.

- Allocation formula changed from proportional to confidence alone to
  proportional to confidence * quality_score per spec §45003. Both
  DefaultBudgetAllocator and ProportionalBudgetAllocator use the new
  weighted formula. With default quality_score=1.0, behavior is backward-
  compatible.

- DetailDepthResolver.resolve() now accepts budget: int parameter per
  DetailDepthResolverProtocol spec (§44803-44814), enabling future
  budget-aware depth resolution decisions.

- Default packer changed from no-op DefaultBudgetPacker to production
  GreedyKnapsackPacker per spec §45013. ACMSPipeline uses lazy import
  to avoid circular dependency; ContextAssemblyPipeline imports directly.

- Added quality_score field to service-layer StrategyCapabilities
  (already present in domain model).

- Updated 5 feature scenarios and 1 Robot helper where GreedyKnapsackPacker
  relevance-based ordering changed fragment output order vs. the old no-op
  packer. Fixed by adjusting test fragment relevance scores to preserve
  expected ordering.

ISSUES CLOSED: #924
2026-03-26 06:19:14 +00:00

393 lines
14 KiB
Python

"""Step definitions for features/acms_service_coverage_boost.feature.
Targets uncovered lines in src/cleveragents/application/services/acms_service.py:
- Line 413: DefaultBudgetAllocator.allocate with empty candidates
- Line 436: DefaultBudgetAllocator.allocate remainder token distribution
- Line 463: DefaultStrategyExecutor.execute with empty allocations
- Line 467: DefaultStrategyExecutor.execute skipping zero-token allocations
- Lines 616-620: ACMSPipeline.__init__ unknown default_strategy ValueError
"""
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
from behave import given, then, when
from behave.runner import Context
from cleveragents.application.services.acms_service import (
ACMSPipeline,
DefaultBudgetAllocator,
DefaultStrategyExecutor,
RecencyStrategy,
RelevanceStrategy,
StrategyCapabilities,
TieredStrategy,
)
from cleveragents.domain.models.core.context_fragment import (
ContextBudget,
ContextFragment,
FragmentProvenance,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_DEFAULT_PROVENANCE = FragmentProvenance(resource_uri="test://coverage-boost")
def _make_frag(
uko_node: str = "test://cov",
content: str = "test",
token_count: int = 10,
relevance_score: float = 0.5,
**kwargs: Any,
) -> ContextFragment:
kwargs.setdefault("provenance", _DEFAULT_PROVENANCE)
return ContextFragment(
uko_node=uko_node,
content=content,
token_count=token_count,
relevance_score=relevance_score,
**kwargs,
)
class _SimpleStrategy:
"""Minimal strategy for testing the executor."""
def __init__(self, name: str = "simple") -> None:
self._name = name
@property
def name(self) -> str:
return self._name
@property
def capabilities(self) -> StrategyCapabilities:
return StrategyCapabilities(quality_score=1.0)
def can_handle(self, request: dict[str, Any]) -> float:
return 0.9
def assemble(
self,
fragments: Sequence[ContextFragment],
budget: ContextBudget,
) -> Sequence[ContextFragment]:
# Just return all fragments that fit.
result: list[ContextFragment] = []
total = 0
for frag in fragments:
if total + frag.token_count <= budget.available_tokens:
result.append(frag)
total += frag.token_count
return result
def explain(self) -> str:
return "Simple test strategy."
# ---------------------------------------------------------------------------
# Given — DefaultBudgetAllocator
# ---------------------------------------------------------------------------
@given("the default budget allocator")
def step_given_default_allocator(context: Context) -> None:
context.allocator = DefaultBudgetAllocator()
# ---------------------------------------------------------------------------
# Given — DefaultStrategyExecutor
# ---------------------------------------------------------------------------
@given("the default strategy executor")
def step_given_default_executor(context: Context) -> None:
context.executor = DefaultStrategyExecutor()
@given("a test strategy and fragments for execution")
def step_given_test_strategy_and_fragments(context: Context) -> None:
context.test_strategy = _SimpleStrategy("test_exec")
context.exec_fragments = [
_make_frag(uko_node="test://a", content="alpha", token_count=10),
_make_frag(uko_node="test://b", content="beta", token_count=20),
]
context.exec_budget = ContextBudget(max_tokens=200, reserved_tokens=0)
# ---------------------------------------------------------------------------
# When — DefaultBudgetAllocator
# ---------------------------------------------------------------------------
@when("I allocate budget {budget:d} to an empty candidate list")
def step_allocate_empty_candidates(context: Context, budget: int) -> None:
"""Exercises line 413: return [] for empty candidates."""
context.alloc_result = context.allocator.allocate([], budget)
@when(
"I allocate budget {budget:d} across two candidates with confidences {c1:g} and {c2:g}"
)
def step_allocate_two_candidates_fractional(
context: Context, budget: int, c1: float, c2: float
) -> None:
"""Exercises line 436: remainder distribution in proportional allocation.
Budget 100 with confidences 0.5 and 0.3 (total 0.8):
raw = [62.5, 37.5], floors = [62, 37], remainder = 1
-> line 436 awards the extra token to the first candidate.
"""
s1 = RelevanceStrategy()
s2 = RecencyStrategy()
context.alloc_result = context.allocator.allocate([(s1, c1), (s2, c2)], budget)
@when(
"I allocate budget {budget:d} across three candidates with confidences {conf_str}"
)
def step_allocate_three_uneven(context: Context, budget: int, conf_str: str) -> None:
"""Exercises line 436 with three candidates and non-zero remainder."""
confidences = [float(c) for c in conf_str.split()]
strategies = [RelevanceStrategy(), RecencyStrategy(), TieredStrategy()]
candidates = [
(s, c) for s, c in zip(strategies[: len(confidences)], confidences, strict=True)
]
context.alloc_result = context.allocator.allocate(candidates, budget)
@when("I allocate budget {budget:d} across three zero-confidence candidates")
def step_allocate_zero_conf_remainder(context: Context, budget: int) -> None:
"""Exercises lines 418-423: zero-confidence equal split with remainder.
Budget 10, 3 candidates, all confidence 0.0:
share = 10 // 3 = 3, remainder = 10 - 9 = 1
-> first candidate gets 3+1=4, others get 3.
"""
strategies = [RelevanceStrategy(), RecencyStrategy(), TieredStrategy()]
candidates = [(s, 0.0) for s in strategies]
context.alloc_result = context.allocator.allocate(candidates, budget)
# ---------------------------------------------------------------------------
# When — DefaultStrategyExecutor
# ---------------------------------------------------------------------------
@when("I execute with an empty allocations list")
def step_execute_empty_allocations(context: Context) -> None:
"""Exercises line 463: return [] for empty allocations."""
dummy_budget = ContextBudget(max_tokens=100, reserved_tokens=0)
context.exec_result = context.executor.execute([], [], dummy_budget)
@when("I execute with a zero-token allocation for the test strategy")
def step_execute_zero_token(context: Context) -> None:
"""Exercises line 467: continue when allocated_tokens <= 0."""
allocations = [(context.test_strategy, 0.8, 0)]
context.exec_result = context.executor.execute(
allocations, context.exec_fragments, context.exec_budget
)
@when("I execute with mixed allocations including zero and positive tokens")
def step_execute_mixed_allocations(context: Context) -> None:
"""Exercises line 467 (skip) and line 472 (process positive)."""
zero_strategy = _SimpleStrategy("zero_strat")
positive_strategy = _SimpleStrategy("pos_strat")
allocations = [
(zero_strategy, 0.5, 0), # should be skipped (line 467)
(positive_strategy, 0.8, 100), # should be processed
]
context.exec_result = context.executor.execute(
allocations, context.exec_fragments, context.exec_budget
)
# ---------------------------------------------------------------------------
# When — ACMSPipeline.__init__ unknown default_strategy
# ---------------------------------------------------------------------------
@when('I create a pipeline with default strategy "{strategy}"')
def step_create_pipeline_unknown_strategy(context: Context, strategy: str) -> None:
"""Exercises lines 616-620: ValueError for unknown default_strategy."""
context.pipeline_error = None
try:
context.test_pipeline = ACMSPipeline(default_strategy=strategy)
except ValueError as exc:
context.pipeline_error = exc
@when("I create a pipeline with an empty default strategy")
def step_create_pipeline_empty_strategy(context: Context) -> None:
"""Exercises lines 616-620: ValueError for empty string default_strategy."""
context.pipeline_error = None
try:
context.test_pipeline = ACMSPipeline(default_strategy="")
except ValueError as exc:
context.pipeline_error = exc
# ---------------------------------------------------------------------------
# When/Given — Full pipeline with multi-candidate selector
# ---------------------------------------------------------------------------
class _MultiCandidateSelector:
"""Returns multiple candidates to force proportional allocation."""
def select(
self,
strategies: Sequence[Any],
request: dict[str, Any],
) -> list[tuple[Any, float]]:
# Return first two strategies with different confidences
# so the allocator exercises the remainder path.
result = []
for s in strategies[:2]:
conf = 0.7 if s.name == "relevance" else 0.3
result.append((s, conf))
return result
@given("a pipeline with a custom selector that returns multiple candidates")
def step_pipeline_multi_candidate_selector(context: Context) -> None:
context.pipeline = ACMSPipeline(strategy_selector=_MultiCandidateSelector())
@given("a context budget with {max_tok:d} max tokens")
def step_budget_simple(context: Context, max_tok: int) -> None:
context.budget = ContextBudget(max_tokens=max_tok, reserved_tokens=0)
@given("simple test fragments totalling {total:d} tokens")
def step_simple_fragments(context: Context, total: int) -> None:
half = total // 2
context.fragments = [
_make_frag(
uko_node="test://frag1",
content="fragment one",
token_count=half,
relevance_score=0.9,
),
_make_frag(
uko_node="test://frag2",
content="fragment two",
token_count=total - half,
relevance_score=0.7,
),
]
@when("I assemble the fragments through the full pipeline")
def step_assemble_full_pipeline(context: Context) -> None:
context.assemble_error = None
try:
context.payload = context.pipeline.assemble(
plan_id="01JQTESTPN00000000000000AA",
fragments=context.fragments,
budget=context.budget,
strategy="relevance",
)
except Exception as exc:
context.assemble_error = exc
# ---------------------------------------------------------------------------
# Then — DefaultBudgetAllocator assertions
# ---------------------------------------------------------------------------
@then("the allocation result should be an empty list")
def step_alloc_result_empty(context: Context) -> None:
assert context.alloc_result == [], (
f"Expected empty list, got {context.alloc_result}"
)
@then("the total allocated tokens should equal exactly {budget:d}")
def step_total_alloc_exact(context: Context, budget: int) -> None:
total = sum(a[2] for a in context.alloc_result)
assert total == budget, (
f"Expected total allocation {budget}, got {total}. "
f"Allocations: {[a[2] for a in context.alloc_result]}"
)
@then("each candidate should receive a positive allocation")
def step_each_positive(context: Context) -> None:
for _strategy, _confidence, tokens in context.alloc_result:
assert tokens > 0, f"Expected positive allocation, got {tokens}"
@then("allocations should be {high:d} or {low:d} tokens each")
def step_allocs_bounded(context: Context, high: int, low: int) -> None:
valid = {high, low}
for _strategy, _confidence, tokens in context.alloc_result:
assert tokens in valid, f"Expected allocation in {valid}, got {tokens}"
# ---------------------------------------------------------------------------
# Then — DefaultStrategyExecutor assertions
# ---------------------------------------------------------------------------
@then("the executor result should be an empty list")
def step_exec_result_empty(context: Context) -> None:
assert list(context.exec_result) == [], (
f"Expected empty result, got {list(context.exec_result)}"
)
@then("the executor result should contain only fragments from the positive allocation")
def step_exec_result_positive_only(context: Context) -> None:
result = list(context.exec_result)
assert len(result) > 0, "Expected non-empty result from positive allocation"
# The positive strategy should have returned fragments
assert len(result) == len(context.exec_fragments), (
f"Expected {len(context.exec_fragments)} fragments, got {len(result)}"
)
# ---------------------------------------------------------------------------
# Then — ACMSPipeline.__init__ error assertions
# ---------------------------------------------------------------------------
@then('the pipeline creation should fail with a ValueError about "{keyword}"')
def step_pipeline_value_error_raised(context: Context, keyword: str) -> None:
assert context.pipeline_error is not None, "Expected ValueError but none was raised"
assert isinstance(context.pipeline_error, ValueError), (
f"Expected ValueError, got {type(context.pipeline_error).__name__}"
)
assert keyword.lower() in str(context.pipeline_error).lower(), (
f"Expected '{keyword}' in error: {context.pipeline_error}"
)
@then("the pipeline error message should list available strategies")
def step_pipeline_error_lists_strategies(context: Context) -> None:
msg = str(context.pipeline_error)
# The error message should mention at least the built-in strategies
for name in ("recency", "relevance", "tiered"):
assert name in msg, f"Expected strategy '{name}' listed in error: {msg}"
# ---------------------------------------------------------------------------
# Then — Full pipeline assertions
# ---------------------------------------------------------------------------
@then("the assembly should succeed without error")
def step_assembly_success(context: Context) -> None:
assert context.assemble_error is None, (
f"Assembly failed with error: {context.assemble_error}"
)
assert context.payload is not None, "Payload is None"