Files
cleveragents-core/features/steps/adaptive_context_strategy_steps.py
T
HAL9000 21ba6dbc9e
CI / load-versions (pull_request) Successful in 14s
CI / push-validation (pull_request) Successful in 24s
CI / lint (pull_request) Successful in 41s
CI / typecheck (pull_request) Successful in 59s
CI / security (pull_request) Successful in 1m12s
CI / quality (pull_request) Successful in 42s
CI / build (pull_request) Successful in 52s
CI / helm (pull_request) Successful in 39s
CI / unit_tests (pull_request) Successful in 5m21s
CI / docker (pull_request) Successful in 1m40s
CI / integration_tests (pull_request) Successful in 8m59s
CI / coverage (pull_request) Successful in 9m44s
CI / status-check (pull_request) Successful in 3s
test(acms): cover adaptive_selector error paths and StrategyWeight validator
Add five behave scenarios exercising the previously-uncovered error
branches in AdaptiveContextSelector (select_strategy and
select_strategies with an unconfigured plan_type) and ContextFusion
(fuse_results and fuse_with_selector with an unconfigured plan_type),
plus a scenario constructing a valid-weight StrategyWeight to cover
the success path of the field_validator.

ISSUES CLOSED: #5255
2026-06-15 04:28:02 -04:00

718 lines
25 KiB
Python

"""Step definitions for adaptive context strategy selector and fusion tests."""
from __future__ import annotations
from typing import Any
from behave import given, then, when
from cleveragents.domain.models.acms.adaptive_selector import (
AdaptiveContextSelector,
AdaptiveStrategyConfig,
ContextFusion,
FusedResult,
PlanType,
StrategyWeight,
)
from cleveragents.domain.models.acms.crp import ContextFragment
from cleveragents.domain.models.acms.strategy import (
BackendSet,
ContextRequest,
ContextStrategy,
PlanContext,
StrategyCapabilities,
)
def _strip_quoted_csv(value: str) -> list[str]:
"""Split a comma-separated quoted-string list and strip outer quotes.
The Gherkin form ``"a", "b", "c"`` is captured by behave as a single
placeholder containing the inner quotes (``a", "b", "c``); split by
``", "`` and stripping leftover quotes recovers the original tokens.
"""
return [item.strip().strip('"') for item in value.split(", ")]
def _table_pairs(table: Any) -> list[tuple[str, str]]:
"""Read a headerless 2-column behave table as ``(key, value)`` pairs.
Behave promotes the first row of a table to ``headings`` automatically.
For the no-header tables used by these scenarios we recover the lost
pair from ``headings`` and then iterate the data rows.
"""
pairs: list[tuple[str, str]] = []
if len(table.headings) >= 2:
pairs.append((table.headings[0], table.headings[1]))
for row in table:
pairs.append((row.cells[0], row.cells[1]))
return pairs
class MockStrategy:
"""Mock strategy for testing."""
def __init__(self, strategy_name: str) -> None:
"""Initialize mock strategy."""
self._name = strategy_name
@property
def name(self) -> str:
"""Return strategy name."""
return self._name
@property
def capabilities(self) -> StrategyCapabilities:
"""Return strategy capabilities."""
return StrategyCapabilities()
def can_handle(
self,
request: ContextRequest,
backends: BackendSet,
) -> float:
"""Return confidence for this request."""
return 1.0
def assemble(
self,
request: ContextRequest,
backends: BackendSet,
budget: int,
plan_context: PlanContext,
) -> list[ContextFragment]:
"""Execute mock strategy."""
return []
def explain(self) -> str:
"""Return explanation."""
return f"Mock strategy: {self._name}"
# Verify MockStrategy satisfies the ContextStrategy protocol
assert isinstance(MockStrategy("test"), ContextStrategy)
class MockStrategyResult:
"""Mock strategy result for testing."""
def __init__(self, ranked_files: list[tuple[str, float]]) -> None:
"""Initialize mock result."""
self.ranked_files = ranked_files
@given("I have an adaptive context selector")
def step_have_selector(context: Any) -> None:
"""Initialize adaptive context selector."""
context.selector = AdaptiveContextSelector()
@given("I have a context fusion engine")
def step_have_fusion(context: Any) -> None:
"""Initialize context fusion engine."""
if not hasattr(context, "selector"):
context.selector = AdaptiveContextSelector()
context.fusion = ContextFusion(context.selector)
@when('I register a strategy named "{name}" with a mock implementation')
def step_register_strategy(context: Any, name: str) -> None:
"""Register a mock strategy."""
strategy = MockStrategy(name)
context.selector.register_strategy(name, strategy)
@then('the strategy "{name}" should be registered')
def step_verify_strategy_registered(context: Any, name: str) -> None:
"""Verify strategy is registered."""
strategies = context.selector.list_registered_strategies()
assert name in strategies, f"Strategy {name} not found in {strategies}"
@then('the list of registered strategies should contain "{name}"')
def step_verify_strategy_in_list(context: Any, name: str) -> None:
"""Verify strategy is in list."""
strategies = context.selector.list_registered_strategies()
assert name in strategies
@when(
'I register configuration for plan type "{plan_type}" with primary strategy "{strategy}"'
)
def step_register_config_simple(context: Any, plan_type: str, strategy: str) -> None:
"""Register configuration for plan type."""
plan_type_enum = PlanType(plan_type)
config = AdaptiveStrategyConfig(
plan_type=plan_type_enum,
primary_strategy=strategy,
)
context.selector.register_config(config)
@then('the configuration for plan type "{plan_type}" should exist')
def step_verify_config_exists(context: Any, plan_type: str) -> None:
"""Verify configuration exists."""
plan_type_enum = PlanType(plan_type)
config = context.selector.get_config(plan_type_enum)
assert config is not None, f"No configuration for {plan_type}"
@then('the primary strategy for "{plan_type}" should be "{strategy}"')
def step_verify_primary_strategy(context: Any, plan_type: str, strategy: str) -> None:
"""Verify primary strategy."""
plan_type_enum = PlanType(plan_type)
config = context.selector.get_config(plan_type_enum)
assert config is not None
assert config.primary_strategy == strategy
@when('I select a strategy for plan type "{plan_type}"')
def step_select_strategy(context: Any, plan_type: str) -> None:
"""Select strategy for plan type."""
plan_type_enum = PlanType(plan_type)
context.selected_strategy = context.selector.select_strategy(plan_type_enum)
@then('the selected strategy should be "{name}"')
def step_verify_selected_strategy(context: Any, name: str) -> None:
"""Verify selected strategy."""
assert context.selected_strategy.name == name
@given('I have registered strategies: "{strategies}"')
def step_register_multiple_strategies(context: Any, strategies: str) -> None:
"""Register multiple strategies."""
for strategy_name in _strip_quoted_csv(strategies):
strategy = MockStrategy(strategy_name)
context.selector.register_strategy(strategy_name, strategy)
@given('I have registered configuration for plan type "{plan_type}" with:')
def step_register_config_with_table(context: Any, plan_type: str) -> None:
"""Register configuration with table data (headerless 2-column table)."""
plan_type_enum = PlanType(plan_type)
config_data: dict[str, Any] = {}
for key, value in _table_pairs(context.table):
if key == "fallback_strategies":
config_data["fallback_strategies"] = [s.strip() for s in value.split(",")]
elif key == "fusion_weights":
weights = {}
for pair in value.split(","):
strategy, weight = pair.strip().split("=")
weights[strategy] = float(weight)
config_data["fusion_weights"] = weights
else:
config_data[key] = value
config = AdaptiveStrategyConfig(
plan_type=plan_type_enum,
primary_strategy=config_data.get("primary_strategy", ""),
fallback_strategies=config_data.get("fallback_strategies", []),
fusion_weights=config_data.get("fusion_weights", {}),
)
context.selector.register_config(config)
@when('I select all strategies for plan type "{plan_type}"')
def step_select_all_strategies(context: Any, plan_type: str) -> None:
"""Select all strategies for plan type."""
plan_type_enum = PlanType(plan_type)
context.selected_strategies = context.selector.select_strategies(plan_type_enum)
@then('I should get {count:d} strategies in order: "{strategies}"')
def step_verify_strategy_order(context: Any, count: int, strategies: str) -> None:
"""Verify strategy order."""
expected = _strip_quoted_csv(strategies)
assert len(context.selected_strategies) == count
for i, expected_name in enumerate(expected):
assert context.selected_strategies[i].name == expected_name
@when('I try to register a strategy named "{name}" again')
def step_try_duplicate_registration(context: Any, name: str) -> None:
"""Try to register duplicate strategy."""
try:
strategy = MockStrategy(name)
context.selector.register_strategy(name, strategy)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about duplicate registration")
def step_verify_duplicate_error(context: Any) -> None:
"""Verify duplicate registration error."""
assert context.error is not None
assert "already registered" in context.error
@when('I try to register configuration with unregistered primary strategy "{strategy}"')
def step_try_unregistered_primary(context: Any, strategy: str) -> None:
"""Try to register config with unregistered primary strategy."""
try:
config = AdaptiveStrategyConfig(
plan_type=PlanType.CODING,
primary_strategy=strategy,
)
context.selector.register_config(config)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about unregistered strategy")
def step_verify_unregistered_error(context: Any) -> None:
"""Verify unregistered strategy error."""
assert context.error is not None
assert "not registered" in context.error
@when(
'I try to register configuration with primary "{primary}" and fallback "{fallback}"'
)
def step_try_unregistered_fallback(context: Any, primary: str, fallback: str) -> None:
"""Try to register config with unregistered fallback."""
try:
config = AdaptiveStrategyConfig(
plan_type=PlanType.CODING,
primary_strategy=primary,
fallback_strategies=[fallback],
)
context.selector.register_config(config)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about unregistered fallback strategy")
def step_verify_unregistered_fallback_error(context: Any) -> None:
"""Verify unregistered fallback error."""
assert context.error is not None
assert "not registered" in context.error
@given("I have strategy results:")
def step_have_strategy_results(context: Any) -> None:
"""Parse strategy results from table."""
context.strategy_results = {}
for row in context.table:
strategy = row["strategy"]
files_str = row["files"]
ranked_files = []
for file_pair in files_str.split(", "):
file_path, score = file_pair.split(":")
ranked_files.append((file_path, float(score)))
context.strategy_results[strategy] = MockStrategyResult(ranked_files)
@when('I fuse the results for plan type "{plan_type}" with equal weights')
def step_fuse_equal_weights(context: Any, plan_type: str) -> None:
"""Fuse results with equal weights."""
plan_type_enum = PlanType(plan_type)
context.fused_result = context.fusion.fuse_results(
plan_type_enum,
context.strategy_results,
)
@when('I fuse the results for plan type "{plan_type}"')
def step_fuse_for_plan_type(context: Any, plan_type: str) -> None:
"""Fuse results using the configured weights for ``plan_type``."""
plan_type_enum = PlanType(plan_type)
context.fused_result = context.fusion.fuse_results(
plan_type_enum,
context.strategy_results,
)
@then("the fused result should have ranked files:")
def step_verify_ranked_files(context: Any) -> None:
"""Verify ranked files in fused result."""
expected_files = {}
for row in context.table:
file_path = row["file"]
score = float(row["score"])
expected_files[file_path] = score
for file_path, expected_score in expected_files.items():
actual_score = context.fused_result.get_file_score(file_path)
assert actual_score is not None, f"File {file_path} not in results"
assert abs(actual_score - expected_score) < 0.0001, (
f"Score mismatch for {file_path}: "
f"expected {expected_score}, got {actual_score}"
)
@when("I fuse the results with custom weights:")
def step_fuse_custom_weights(context: Any) -> None:
"""Fuse results with custom weights (headerless 2-column table)."""
weights: dict[str, float] = {}
for strategy, weight_str in _table_pairs(context.table):
weights[strategy] = float(weight_str)
plan_type_enum = PlanType.CODING
context.fused_result = context.fusion.fuse_results(
plan_type_enum,
context.strategy_results,
custom_weights=weights,
)
@given("I have a fused result with ranked files:")
def step_have_fused_result(context: Any) -> None:
"""Create a fused result from table."""
ranked_files = []
strategy_contributions: dict[str, list[tuple[str, float]]] = {}
for row in context.table:
file_path = row["file"]
score = float(row["score"])
ranked_files.append((file_path, score))
context.fused_result = FusedResult(
ranked_files=ranked_files,
strategy_contributions=strategy_contributions,
)
@when("I get the top {count:d} files")
def step_get_top_files(context: Any, count: int) -> None:
"""Get top N files from fused result."""
context.top_files = context.fused_result.get_top_files(limit=count)
@then('I should get: "{files}"')
def step_verify_top_files(context: Any, files: str) -> None:
"""Verify top files."""
expected = _strip_quoted_csv(files)
assert context.top_files == expected
@when('I get the score for "{file_path}"')
def step_get_file_score(context: Any, file_path: str) -> None:
"""Get score for a file."""
context.file_score = context.fused_result.get_file_score(file_path)
@then("the file score should be {value}")
def step_verify_file_score(context: Any, value: str) -> None:
"""Verify file score."""
if value == "None":
assert context.file_score is None
else:
expected = float(value)
assert abs(context.file_score - expected) < 0.0001
@when("I try to fuse with empty results")
def step_try_fuse_empty(context: Any) -> None:
"""Try to fuse with empty results."""
try:
plan_type_enum = PlanType.CODING
context.fusion.fuse_results(plan_type_enum, {})
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about no results provided")
def step_verify_no_results_error(context: Any) -> None:
"""Verify no results error."""
assert context.error is not None
assert "No strategy results" in context.error
@given('I have strategy results with "{strategy1}" and "{strategy2}"')
def step_have_two_strategies(context: Any, strategy1: str, strategy2: str) -> None:
"""Create strategy results with two strategies."""
context.strategy_results = {
strategy1: MockStrategyResult([(f"{strategy1}_file.py", 0.8)]),
strategy2: MockStrategyResult([(f"{strategy2}_file.py", 0.6)]),
}
@when('I try to fuse with negative weight for "{strategy}"')
def step_try_negative_weight(context: Any, strategy: str) -> None:
"""Try to fuse with negative weight."""
try:
plan_type_enum = PlanType.CODING
context.fusion.fuse_results(
plan_type_enum,
context.strategy_results,
custom_weights={strategy: -0.5},
)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about invalid weight")
def step_verify_invalid_weight_error(context: Any) -> None:
"""Verify invalid weight error."""
assert context.error is not None
assert "positive" in context.error.lower()
@when("I normalize weights: {weights}")
def step_normalize_weights(context: Any, weights: str) -> None:
"""Normalize weights."""
weight_dict = {}
for pair in weights.split(", "):
strategy, weight = pair.split("=")
weight_dict[strategy] = float(weight)
context.normalized = context.fusion._normalize_weights(
weight_dict,
weight_dict.keys(),
)
@then("the normalized weights should be:")
def step_verify_normalized_weights(context: Any) -> None:
"""Verify normalized weights (headerless 2-column table)."""
for strategy, weight_str in _table_pairs(context.table):
expected = float(weight_str)
actual = context.normalized[strategy]
assert abs(actual - expected) < 0.0001, (
f"Weight mismatch for {strategy}: expected {expected}, got {actual}"
)
@then("the fusion metadata should contain:")
def step_verify_fusion_metadata(context: Any) -> None:
"""Verify fusion metadata (headerless 2-column table)."""
for key, value in _table_pairs(context.table):
if key in ("num_strategies", "num_files"):
expected: Any = int(value)
else:
expected = value
actual = context.fused_result.fusion_metadata[key]
assert actual == expected, (
f"Metadata mismatch for {key}: expected {expected}, got {actual}"
)
@given('I have registered configuration for plan types: "{plan_types}"')
def step_register_multiple_configs(context: Any, plan_types: str) -> None:
"""Register configurations for multiple plan types."""
for plan_type_str in _strip_quoted_csv(plan_types):
plan_type_enum = PlanType(plan_type_str)
strategy = MockStrategy(f"{plan_type_str}_strategy")
context.selector.register_strategy(f"{plan_type_str}_strategy", strategy)
config = AdaptiveStrategyConfig(
plan_type=plan_type_enum,
primary_strategy=f"{plan_type_str}_strategy",
)
context.selector.register_config(config)
@when("I list all configured plan types")
def step_list_plan_types(context: Any) -> None:
"""List all configured plan types."""
context.plan_types = context.selector.list_configured_plan_types()
@then('I should get plan types: "{plan_types}"')
def step_verify_plan_types(context: Any, plan_types: str) -> None:
"""Verify plan types."""
expected = [PlanType(pt) for pt in _strip_quoted_csv(plan_types)]
assert context.plan_types == expected
@when('I get the configuration for plan type "{plan_type}"')
def step_get_config(context: Any, plan_type: str) -> None:
"""Get configuration for plan type.
Stored on ``fetched_config`` because behave reserves ``context.config``
for its own runtime configuration object.
"""
plan_type_enum = PlanType(plan_type)
context.fetched_config = context.selector.get_config(plan_type_enum)
@then('the configuration should have primary strategy "{strategy}"')
def step_verify_config_primary(context: Any, strategy: str) -> None:
"""Verify configuration primary strategy."""
assert context.fetched_config is not None
assert context.fetched_config.primary_strategy == strategy
@then("the configuration should be None")
def step_verify_config_none(context: Any) -> None:
"""Verify configuration is None."""
assert context.fetched_config is None
@when("I fuse with selector configuration weights")
def step_fuse_selector_weights(context: Any) -> None:
"""Fuse with selector configuration weights."""
plan_type_enum = PlanType.CODING
context.fused_result = context.fusion.fuse_with_selector(
plan_type_enum,
context.strategy_results,
)
@then('the fused result should have file "{file_path}" with score {score}')
def step_verify_file_score_exact(context: Any, file_path: str, score: str) -> None:
"""Verify exact file score."""
expected = float(score)
actual = context.fused_result.get_file_score(file_path)
assert actual is not None
assert abs(actual - expected) < 0.0001
@given('I have strategy results where "{strategy}" has no ranked_files attribute')
def step_have_result_without_ranked_files(context: Any, strategy: str) -> None:
"""Create strategy result without ranked_files."""
context.strategy_results = {
"semantic": MockStrategyResult([("file1.py", 0.8)]),
strategy: object(), # Object without ranked_files
}
@then("the fusion should skip the strategy without ranked_files")
def step_verify_skip_no_ranked_files(context: Any) -> None:
"""Verify strategy without ranked_files is skipped."""
assert context.fused_result is not None
assert len(context.fused_result.ranked_files) > 0
@when("I try to create a strategy weight with negative weight")
def step_try_negative_strategy_weight(context: Any) -> None:
"""Try to create strategy weight with negative weight."""
try:
StrategyWeight(strategy_name="test", weight=-0.5)
context.error = None
except ValueError as e:
context.error = str(e)
@when("I try to create adaptive strategy config without primary strategy")
def step_try_config_no_primary(context: Any) -> None:
"""Try to create config without primary strategy."""
try:
AdaptiveStrategyConfig(
plan_type=PlanType.CODING,
primary_strategy="",
)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about missing primary strategy")
def step_verify_missing_primary_error(context: Any) -> None:
"""Verify missing primary strategy error."""
assert context.error is not None
assert "primary_strategy" in context.error
@then('I should have plan types: "{plan_types}"')
def step_verify_plan_type_enum(context: Any, plan_types: str) -> None:
"""Verify plan type enumeration."""
expected = _strip_quoted_csv(plan_types)
actual = [pt.value for pt in PlanType]
assert actual == expected, f"PlanType mismatch: expected {expected}, got {actual}"
@given('I have registered a strategy named "{name}"')
def step_have_registered_strategy(context: Any, name: str) -> None:
"""Register a strategy by name."""
strategy = MockStrategy(name)
context.selector.register_strategy(name, strategy)
@when('I try to select a strategy for unconfigured plan type "{plan_type}"')
def step_try_select_unconfigured(context: Any, plan_type: str) -> None:
"""Try to select a strategy when no configuration exists for plan_type."""
try:
context.selector.select_strategy(PlanType(plan_type))
context.error = None
except ValueError as e:
context.error = str(e)
@when('I try to select all strategies for unconfigured plan type "{plan_type}"')
def step_try_select_all_unconfigured(context: Any, plan_type: str) -> None:
"""Try to select all strategies when no configuration exists for plan_type."""
try:
context.selector.select_strategies(PlanType(plan_type))
context.error = None
except ValueError as e:
context.error = str(e)
@when('I try to fuse results for unconfigured plan type "{plan_type}"')
def step_try_fuse_unconfigured(context: Any, plan_type: str) -> None:
"""Try to fuse results when no configuration exists for plan_type."""
try:
context.fusion.fuse_results(PlanType(plan_type), context.strategy_results)
context.error = None
except ValueError as e:
context.error = str(e)
@when('I try to fuse with selector for unconfigured plan type "{plan_type}"')
def step_try_fuse_with_selector_unconfigured(context: Any, plan_type: str) -> None:
"""Try fuse_with_selector when no configuration exists for plan_type."""
try:
context.fusion.fuse_with_selector(PlanType(plan_type), context.strategy_results)
context.error = None
except ValueError as e:
context.error = str(e)
@then("I should get an error about no configuration")
def step_verify_no_configuration_error(context: Any) -> None:
"""Verify error mentions missing plan-type configuration."""
assert context.error is not None
assert "No configuration" in context.error
@when('I create a strategy weight named "{name}" with weight {weight:f}')
def step_create_valid_strategy_weight(context: Any, name: str, weight: float) -> None:
"""Create a StrategyWeight with a valid positive weight."""
context.strategy_weight = StrategyWeight(strategy_name=name, weight=weight)
@then('the strategy weight should have name "{name}" and weight {weight:f}')
def step_verify_strategy_weight(context: Any, name: str, weight: float) -> None:
"""Verify created StrategyWeight fields."""
assert context.strategy_weight.strategy_name == name
assert abs(context.strategy_weight.weight - weight) < 0.0001
@given(
'I have registered configuration for plan type "{plan_type}" with primary strategy "{strategy}"'
)
def step_have_registered_config(context: Any, plan_type: str, strategy: str) -> None:
"""Register configuration for plan type, auto-registering the strategy."""
plan_type_enum = PlanType(plan_type)
if strategy not in context.selector.list_registered_strategies():
context.selector.register_strategy(strategy, MockStrategy(strategy))
config = AdaptiveStrategyConfig(
plan_type=plan_type_enum,
primary_strategy=strategy,
)
context.selector.register_config(config)
@given('I have registered configuration for plan type "{plan_type}"')
def step_have_registered_config_no_strategy(context: Any, plan_type: str) -> None:
"""Register configuration for plan type with a default strategy."""
plan_type_enum = PlanType(plan_type)
strategy_name = f"{plan_type}_default"
strategy = MockStrategy(strategy_name)
context.selector.register_strategy(strategy_name, strategy)
config = AdaptiveStrategyConfig(
plan_type=plan_type_enum,
primary_strategy=strategy_name,
)
context.selector.register_config(config)