freemo
487e16a9f0
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 13s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 32s
CI / typecheck (pull_request) Successful in 33s
CI / unit_tests (pull_request) Successful in 2m7s
CI / docker (pull_request) Successful in 37s
CI / integration_tests (pull_request) Successful in 2m56s
CI / coverage (pull_request) Successful in 4m26s
CI / benchmark-regression (pull_request) Successful in 28m12s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 15s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 31s
CI / typecheck (push) Successful in 34s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m4s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 3m7s
CI / coverage (push) Successful in 5m4s
CI / benchmark-publish (push) Has been cancelled
Implement the first three built-in context strategies for the ACMS v1 context assembly pipeline: 1. SimpleKeywordStrategy (quality 0.3) - Keyword matching on fragment content with word-density fallback. Universal fallback strategy. 2. SemanticEmbeddingStrategy (quality 0.6) - Jaccard word-overlap similarity scoring between query and fragment content. 3. BreadthDepthNavigatorStrategy (quality 0.85) - UKO node hierarchy navigation prioritising fragments near focus nodes with higher detail depths. Primary strategy for code projects. All strategies implement the v1 ContextStrategy Protocol from acms_service.py and can be registered with ACMSPipeline via register_strategy(). Includes: - 28 Behave BDD scenarios covering ranking, budget, capabilities, can_handle confidence, explain, empty input, and pipeline registration - 9 Robot Framework integration tests - ASV benchmarks at 10/100/1000 fragment scales for all 3 strategies - Vulture whitelist entries for public API symbols - 100% coverage on context_strategies.py ISSUES CLOSED: #541
137 lines
4.9 KiB
Python
137 lines
4.9 KiB
Python
"""ASV benchmarks for built-in context strategies batch 1.
|
|
|
|
Measures the performance of:
|
|
- SimpleKeywordStrategy.assemble at varying fragment counts
|
|
- SemanticEmbeddingStrategy.assemble at varying fragment counts
|
|
- BreadthDepthNavigatorStrategy.assemble at varying fragment counts
|
|
- can_handle() call overhead for each strategy
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.application.services.context_strategies import ( # noqa: E402
|
|
BreadthDepthNavigatorStrategy,
|
|
SemanticEmbeddingStrategy,
|
|
SimpleKeywordStrategy,
|
|
)
|
|
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
|
ContextBudget,
|
|
ContextFragment,
|
|
FragmentProvenance,
|
|
)
|
|
|
|
_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://default")
|
|
|
|
|
|
def _make_fragments(count: int) -> list[ContextFragment]:
|
|
"""Build *count* test fragments with distinct URIs and content."""
|
|
frags: list[ContextFragment] = []
|
|
for i in range(count):
|
|
frags.append(
|
|
ContextFragment(
|
|
uko_node=f"project://app/mod{i}.py",
|
|
content=f"Module {i} with async database connection handler pool",
|
|
token_count=20,
|
|
detail_depth=min(i % 10, 9),
|
|
relevance_score=round(0.1 + (i % 9) * 0.1, 2),
|
|
provenance=_DEFAULT_PROV,
|
|
)
|
|
)
|
|
return frags
|
|
|
|
|
|
class SimpleKeywordSuite:
|
|
"""Benchmark SimpleKeywordStrategy assemble throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self.strategy = SimpleKeywordStrategy()
|
|
self.strategy.set_query("async database connection")
|
|
self.budget = ContextBudget(max_tokens=50_000, reserved_tokens=0)
|
|
self.frags_10 = _make_fragments(10)
|
|
self.frags_100 = _make_fragments(100)
|
|
self.frags_1000 = _make_fragments(1000)
|
|
|
|
def time_assemble_10_fragments(self) -> None:
|
|
"""Assemble 10 fragments with SimpleKeywordStrategy."""
|
|
self.strategy.assemble(self.frags_10, self.budget)
|
|
|
|
def time_assemble_100_fragments(self) -> None:
|
|
"""Assemble 100 fragments with SimpleKeywordStrategy."""
|
|
self.strategy.assemble(self.frags_100, self.budget)
|
|
|
|
def time_assemble_1000_fragments(self) -> None:
|
|
"""Assemble 1000 fragments with SimpleKeywordStrategy."""
|
|
self.strategy.assemble(self.frags_1000, self.budget)
|
|
|
|
def time_can_handle(self) -> None:
|
|
"""Benchmark can_handle call overhead."""
|
|
self.strategy.can_handle({"query": "test search"})
|
|
|
|
|
|
class SemanticEmbeddingSuite:
|
|
"""Benchmark SemanticEmbeddingStrategy assemble throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self.strategy = SemanticEmbeddingStrategy()
|
|
self.strategy.set_query("async database connection")
|
|
self.budget = ContextBudget(max_tokens=50_000, reserved_tokens=0)
|
|
self.frags_10 = _make_fragments(10)
|
|
self.frags_100 = _make_fragments(100)
|
|
self.frags_1000 = _make_fragments(1000)
|
|
|
|
def time_assemble_10_fragments(self) -> None:
|
|
"""Assemble 10 fragments with SemanticEmbeddingStrategy."""
|
|
self.strategy.assemble(self.frags_10, self.budget)
|
|
|
|
def time_assemble_100_fragments(self) -> None:
|
|
"""Assemble 100 fragments with SemanticEmbeddingStrategy."""
|
|
self.strategy.assemble(self.frags_100, self.budget)
|
|
|
|
def time_assemble_1000_fragments(self) -> None:
|
|
"""Assemble 1000 fragments with SemanticEmbeddingStrategy."""
|
|
self.strategy.assemble(self.frags_1000, self.budget)
|
|
|
|
def time_can_handle(self) -> None:
|
|
"""Benchmark can_handle call overhead."""
|
|
self.strategy.can_handle({"query": "test search"})
|
|
|
|
|
|
class BreadthDepthNavigatorSuite:
|
|
"""Benchmark BreadthDepthNavigatorStrategy assemble throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self.strategy = BreadthDepthNavigatorStrategy()
|
|
self.strategy.set_focus(["project://app/mod0.py"])
|
|
self.budget = ContextBudget(max_tokens=50_000, reserved_tokens=0)
|
|
self.frags_10 = _make_fragments(10)
|
|
self.frags_100 = _make_fragments(100)
|
|
self.frags_1000 = _make_fragments(1000)
|
|
|
|
def time_assemble_10_fragments(self) -> None:
|
|
"""Assemble 10 fragments with BreadthDepthNavigatorStrategy."""
|
|
self.strategy.assemble(self.frags_10, self.budget)
|
|
|
|
def time_assemble_100_fragments(self) -> None:
|
|
"""Assemble 100 fragments with BreadthDepthNavigatorStrategy."""
|
|
self.strategy.assemble(self.frags_100, self.budget)
|
|
|
|
def time_assemble_1000_fragments(self) -> None:
|
|
"""Assemble 1000 fragments with BreadthDepthNavigatorStrategy."""
|
|
self.strategy.assemble(self.frags_1000, self.budget)
|
|
|
|
def time_can_handle(self) -> None:
|
|
"""Benchmark can_handle call overhead."""
|
|
self.strategy.can_handle({"focus": ["project://app/mod0.py"]})
|