487e16a9f0
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 13s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 32s
CI / typecheck (pull_request) Successful in 33s
CI / unit_tests (pull_request) Successful in 2m7s
CI / docker (pull_request) Successful in 37s
CI / integration_tests (pull_request) Successful in 2m56s
CI / coverage (pull_request) Successful in 4m26s
CI / benchmark-regression (pull_request) Successful in 28m12s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 31s
CI / typecheck (push) Successful in 34s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m4s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m7s
CI / coverage (push) Successful in 5m4s
CI / benchmark-publish (push) Has been cancelled
Implement the first three built-in context strategies for the ACMS v1 context assembly pipeline: 1. SimpleKeywordStrategy (quality 0.3) - Keyword matching on fragment content with word-density fallback. Universal fallback strategy. 2. SemanticEmbeddingStrategy (quality 0.6) - Jaccard word-overlap similarity scoring between query and fragment content. 3. BreadthDepthNavigatorStrategy (quality 0.85) - UKO node hierarchy navigation prioritising fragments near focus nodes with higher detail depths. Primary strategy for code projects. All strategies implement the v1 ContextStrategy Protocol from acms_service.py and can be registered with ACMSPipeline via register_strategy(). Includes: - 28 Behave BDD scenarios covering ranking, budget, capabilities, can_handle confidence, explain, empty input, and pipeline registration - 9 Robot Framework integration tests - ASV benchmarks at 10/100/1000 fragment scales for all 3 strategies - Vulture whitelist entries for public API symbols - 100% coverage on context_strategies.py ISSUES CLOSED: #541
315 lines
11 KiB
Python
315 lines
11 KiB
Python
"""Step definitions for ``features/context_strategies.feature``.
|
|
|
|
Covers the first batch of built-in context strategies:
|
|
|
|
* **SimpleKeywordStrategy** — keyword matching / word-density fallback
|
|
* **SemanticEmbeddingStrategy** — Jaccard word-overlap similarity
|
|
* **BreadthDepthNavigatorStrategy** — UKO hierarchy navigation
|
|
|
|
Also covers pipeline registration of all three strategies.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.application.services.acms_service import ACMSPipeline
|
|
from cleveragents.application.services.context_strategies import (
|
|
BreadthDepthNavigatorStrategy,
|
|
SemanticEmbeddingStrategy,
|
|
SimpleKeywordStrategy,
|
|
)
|
|
from cleveragents.domain.models.core.context_fragment import (
|
|
ContextBudget,
|
|
ContextFragment,
|
|
FragmentProvenance,
|
|
)
|
|
|
|
__all__: list[str] = []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_DEFAULT_PROVENANCE = FragmentProvenance(resource_uri="test://default")
|
|
|
|
|
|
def _make_fragment(
|
|
uko_node: str,
|
|
content: str,
|
|
score: float,
|
|
tokens: int,
|
|
depth: int,
|
|
) -> ContextFragment:
|
|
"""Build a ``ContextFragment`` for test purposes."""
|
|
return ContextFragment(
|
|
uko_node=uko_node,
|
|
content=content,
|
|
relevance_score=score,
|
|
token_count=tokens,
|
|
detail_depth=depth,
|
|
provenance=FragmentProvenance(resource_uri=uko_node),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a SimpleKeywordStrategy with query "{query}"')
|
|
def step_simple_keyword_with_query(context: Context, query: str) -> None:
|
|
strategy = SimpleKeywordStrategy()
|
|
strategy.set_query(query)
|
|
context.strategy = strategy
|
|
|
|
|
|
@given("a SimpleKeywordStrategy without query")
|
|
def step_simple_keyword_no_query(context: Context) -> None:
|
|
context.strategy = SimpleKeywordStrategy()
|
|
|
|
|
|
@given('a SemanticEmbeddingStrategy with query "{query}"')
|
|
def step_semantic_embedding_with_query(context: Context, query: str) -> None:
|
|
strategy = SemanticEmbeddingStrategy()
|
|
strategy.set_query(query)
|
|
context.strategy = strategy
|
|
|
|
|
|
@given("a SemanticEmbeddingStrategy without query")
|
|
def step_semantic_embedding_no_query(context: Context) -> None:
|
|
context.strategy = SemanticEmbeddingStrategy()
|
|
|
|
|
|
@given('a BreadthDepthNavigatorStrategy with focus "{focus}"')
|
|
def step_breadth_depth_with_focus(context: Context, focus: str) -> None:
|
|
strategy = BreadthDepthNavigatorStrategy()
|
|
strategy.set_focus([focus])
|
|
context.strategy = strategy
|
|
|
|
|
|
@given("a BreadthDepthNavigatorStrategy without focus")
|
|
def step_breadth_depth_no_focus(context: Context) -> None:
|
|
context.strategy = BreadthDepthNavigatorStrategy()
|
|
|
|
|
|
@given("the following strategy fragments:")
|
|
def step_strategy_fragments_table(context: Context) -> None:
|
|
context.strategy_fragments = []
|
|
for row in context.table:
|
|
frag = _make_fragment(
|
|
uko_node=row["uko_node"],
|
|
content=row["content"],
|
|
score=float(row["score"]),
|
|
tokens=int(row["tokens"]),
|
|
depth=int(row["depth"]),
|
|
)
|
|
context.strategy_fragments.append(frag)
|
|
|
|
|
|
@given("an empty strategy fragment list")
|
|
def step_empty_strategy_fragments(context: Context) -> None:
|
|
context.strategy_fragments = []
|
|
|
|
|
|
@given(
|
|
"a strategy budget with max_tokens {max_tokens:d} and reserved_tokens {reserved:d}"
|
|
)
|
|
def step_strategy_budget(context: Context, max_tokens: int, reserved: int) -> None:
|
|
context.strategy_budget = ContextBudget(
|
|
max_tokens=max_tokens, reserved_tokens=reserved
|
|
)
|
|
|
|
|
|
@given("an ACMS pipeline for strategy tests")
|
|
def step_pipeline_for_strategies(context: Context) -> None:
|
|
context.pipeline = ACMSPipeline()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I assemble with the SimpleKeywordStrategy")
|
|
def step_assemble_simple_keyword(context: Context) -> None:
|
|
context.strategy_result = list(
|
|
context.strategy.assemble(context.strategy_fragments, context.strategy_budget)
|
|
)
|
|
|
|
|
|
@when("I assemble with the SemanticEmbeddingStrategy")
|
|
def step_assemble_semantic_embedding(context: Context) -> None:
|
|
context.strategy_result = list(
|
|
context.strategy.assemble(context.strategy_fragments, context.strategy_budget)
|
|
)
|
|
|
|
|
|
@when("I assemble with the BreadthDepthNavigatorStrategy")
|
|
def step_assemble_breadth_depth(context: Context) -> None:
|
|
context.strategy_result = list(
|
|
context.strategy.assemble(context.strategy_fragments, context.strategy_budget)
|
|
)
|
|
|
|
|
|
@when('I check can_handle on SimpleKeywordStrategy with query "{query}"')
|
|
def step_can_handle_simple_keyword(context: Context, query: str) -> None:
|
|
request: dict[str, Any] = {"query": query}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when('I check can_handle on SemanticEmbeddingStrategy with query "{query}"')
|
|
def step_can_handle_semantic_with_query(context: Context, query: str) -> None:
|
|
request: dict[str, Any] = {"query": query}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when("I check can_handle on SemanticEmbeddingStrategy without query")
|
|
def step_can_handle_semantic_no_query(context: Context) -> None:
|
|
request: dict[str, Any] = {}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when("I check can_handle on BreadthDepthNavigator with focus")
|
|
def step_can_handle_breadth_depth_with_focus(context: Context) -> None:
|
|
request: dict[str, Any] = {"focus": ["project://app/io.py"]}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when("I check can_handle on BreadthDepthNavigator with string focus")
|
|
def step_can_handle_breadth_depth_string_focus(context: Context) -> None:
|
|
request: dict[str, Any] = {"focus": "project://app/io.py"}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when("I check can_handle on BreadthDepthNavigator without focus")
|
|
def step_can_handle_breadth_depth_no_focus(context: Context) -> None:
|
|
request: dict[str, Any] = {}
|
|
context.confidence = context.strategy.can_handle(request)
|
|
|
|
|
|
@when("I register SimpleKeywordStrategy with the pipeline")
|
|
def step_register_simple_keyword(context: Context) -> None:
|
|
context.pipeline.register_strategy("simple-keyword", SimpleKeywordStrategy())
|
|
|
|
|
|
@when("I register all batch 1 strategies with the pipeline")
|
|
def step_register_all_batch1(context: Context) -> None:
|
|
context.pipeline.register_strategy("simple-keyword", SimpleKeywordStrategy())
|
|
context.pipeline.register_strategy(
|
|
"semantic-embedding", SemanticEmbeddingStrategy()
|
|
)
|
|
context.pipeline.register_strategy(
|
|
"breadth-depth-navigator", BreadthDepthNavigatorStrategy()
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('the first result fragment should have uko_node "{uko_node}"')
|
|
def step_first_result_uko_node(context: Context, uko_node: str) -> None:
|
|
assert len(context.strategy_result) > 0, "Expected at least one result fragment"
|
|
actual = context.strategy_result[0].uko_node
|
|
assert actual == uko_node, (
|
|
f"Expected first fragment uko_node '{uko_node}', got '{actual}'"
|
|
)
|
|
|
|
|
|
@then("{count:d} fragments should be returned by strategy")
|
|
def step_fragment_count(context: Context, count: int) -> None:
|
|
actual = len(context.strategy_result)
|
|
assert actual == count, f"Expected {count} fragments, got {actual}"
|
|
|
|
|
|
@then("the strategy confidence should be {expected:g}")
|
|
def step_strategy_confidence(context: Context, expected: float) -> None:
|
|
actual = context.confidence
|
|
assert abs(actual - expected) < 1e-6, (
|
|
f"Expected strategy confidence {expected}, got {actual}"
|
|
)
|
|
|
|
|
|
@then("the SimpleKeywordStrategy should not support semantic search")
|
|
def step_simple_keyword_no_semantic(context: Context) -> None:
|
|
caps = context.strategy.capabilities
|
|
assert not caps.supports_semantic_search, (
|
|
"SimpleKeywordStrategy should NOT support semantic search"
|
|
)
|
|
|
|
|
|
@then('the SimpleKeywordStrategy name should be "{name}"')
|
|
def step_simple_keyword_name(context: Context, name: str) -> None:
|
|
actual = context.strategy.name
|
|
assert actual == name, f"Expected name '{name}', got '{actual}'"
|
|
|
|
|
|
@then("the SemanticEmbeddingStrategy should support semantic search")
|
|
def step_semantic_supports_search(context: Context) -> None:
|
|
caps = context.strategy.capabilities
|
|
assert caps.supports_semantic_search, (
|
|
"SemanticEmbeddingStrategy should support semantic search"
|
|
)
|
|
|
|
|
|
@then('the SemanticEmbeddingStrategy name should be "{name}"')
|
|
def step_semantic_name(context: Context, name: str) -> None:
|
|
actual = context.strategy.name
|
|
assert actual == name, f"Expected name '{name}', got '{actual}'"
|
|
|
|
|
|
@then("the BreadthDepthNavigatorStrategy should support graph navigation")
|
|
def step_breadth_depth_supports_graph(context: Context) -> None:
|
|
caps = context.strategy.capabilities
|
|
assert caps.supports_graph_navigation, (
|
|
"BreadthDepthNavigatorStrategy should support graph navigation"
|
|
)
|
|
|
|
|
|
@then('the BreadthDepthNavigatorStrategy name should be "{name}"')
|
|
def step_breadth_depth_name(context: Context, name: str) -> None:
|
|
actual = context.strategy.name
|
|
assert actual == name, f"Expected name '{name}', got '{actual}'"
|
|
|
|
|
|
@then('the SimpleKeywordStrategy explain should contain "{text}"')
|
|
def step_simple_keyword_explain(context: Context, text: str) -> None:
|
|
explanation = context.strategy.explain()
|
|
assert text.lower() in explanation.lower(), (
|
|
f"Expected explain to contain '{text}', got: {explanation}"
|
|
)
|
|
|
|
|
|
@then('the SemanticEmbeddingStrategy explain should contain "{text}"')
|
|
def step_semantic_explain(context: Context, text: str) -> None:
|
|
explanation = context.strategy.explain()
|
|
assert text.lower() in explanation.lower(), (
|
|
f"Expected explain to contain '{text}', got: {explanation}"
|
|
)
|
|
|
|
|
|
@then('the BreadthDepthNavigatorStrategy explain should contain "{text}"')
|
|
def step_breadth_depth_explain(context: Context, text: str) -> None:
|
|
explanation = context.strategy.explain()
|
|
assert text.lower() in explanation.lower(), (
|
|
f"Expected explain to contain '{text}', got: {explanation}"
|
|
)
|
|
|
|
|
|
@then('the pipeline should have strategy "{name}"')
|
|
def step_pipeline_has_strategy(context: Context, name: str) -> None:
|
|
strategies = context.pipeline.BUILTIN_STRATEGIES
|
|
registered = context.pipeline._strategies
|
|
has_builtin = name in strategies
|
|
has_registered = name in registered
|
|
assert has_builtin or has_registered, (
|
|
f"Pipeline does not have strategy '{name}'. "
|
|
f"Builtins: {list(strategies.keys())}, Registered: {list(registered.keys())}"
|
|
)
|