freemo
f2232eec09
CI / push-validation (pull_request) Successful in 32s
CI / helm (pull_request) Successful in 42s
CI / build (pull_request) Successful in 45s
CI / lint (pull_request) Successful in 1m13s
CI / quality (pull_request) Successful in 1m14s
CI / typecheck (pull_request) Successful in 1m27s
CI / security (pull_request) Successful in 1m28s
CI / unit_tests (pull_request) Successful in 7m6s
CI / docker (pull_request) Successful in 1m53s
CI / integration_tests (pull_request) Successful in 21m26s
CI / coverage (pull_request) Successful in 13m44s
CI / status-check (pull_request) Successful in 3s
- Updated SkeletonCompressorService.compress() to accept (fragments: tuple[ContextFragment, ...], skeleton_budget: int) -> tuple[ContextFragment, ...], matching the SkeletonCompressor protocol - Removed skeleton_ratio and CompressionResult from the public API - Added @runtime_checkable to SkeletonCompressor protocol in acms_service.py - Added structural subtype assertion at module level to prevent future protocol drift - Rewrote all Behave feature scenarios and step definitions to use skeleton_budget - Updated benchmarks and robot helpers to use absolute token budgets - Removed CompressionResult export from services __init__.py and vulture_whitelist.py - The depth_breadth_projection.py call site already correctly computed and passed skeleton_budget ISSUES CLOSED: #2925
113 lines
3.8 KiB
Python
113 lines
3.8 KiB
Python
"""ASV benchmarks for skeleton compressor overhead.
|
|
|
|
Measures the time to compress context fragments at various budgets
|
|
and fragment counts. The benchmark covers the hot path that runs
|
|
during subplan context inheritance.
|
|
|
|
The caller is responsible for converting a skeleton_ratio to an
|
|
absolute skeleton_budget before invoking compress(). These benchmarks
|
|
use representative absolute token budgets derived from typical ratios
|
|
applied to the fragment set sizes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.application.services.skeleton_compressor import ( # noqa: E402
|
|
SkeletonCompressorService,
|
|
)
|
|
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
|
ContextFragment,
|
|
FragmentProvenance,
|
|
)
|
|
|
|
# Default provenance for skeleton benchmark fragments.
|
|
_SKEL_PROV = FragmentProvenance(resource_uri="bench-skeleton://default")
|
|
|
|
|
|
def _build_fragments(count: int, tokens_each: int = 100) -> tuple[ContextFragment, ...]:
|
|
"""Generate *count* fragments for benchmarking."""
|
|
return tuple(
|
|
ContextFragment(
|
|
fragment_id=f"bench-{i:05d}",
|
|
uko_node=f"bench-skeleton://file/{i}",
|
|
content="x" * tokens_each,
|
|
token_count=tokens_each,
|
|
relevance_score=round(1.0 - (i / max(count, 1)), 4),
|
|
provenance=_SKEL_PROV,
|
|
metadata=(
|
|
{"source_decision_id": f"01HX{'B' * 22}{i:02d}"} if i < 99 else {}
|
|
),
|
|
)
|
|
for i in range(count)
|
|
)
|
|
|
|
|
|
class SkeletonCompressorSmallSuite:
|
|
"""Benchmark compression of a small fragment set (10 fragments, 1000 tokens)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(10) # 1000 total tokens
|
|
|
|
def time_compress_budget_full(self) -> None:
|
|
"""No compression (pass-through) — budget equals total tokens."""
|
|
self._svc.compress(self._fragments, 1000)
|
|
|
|
def time_compress_budget_70pct(self) -> None:
|
|
"""Default-ratio-equivalent compression (budget = 70% of total)."""
|
|
self._svc.compress(self._fragments, 700)
|
|
|
|
def time_compress_budget_50pct(self) -> None:
|
|
"""Moderate compression (budget = 50% of total)."""
|
|
self._svc.compress(self._fragments, 500)
|
|
|
|
def time_compress_budget_zero(self) -> None:
|
|
"""Maximum compression — empty result."""
|
|
self._svc.compress(self._fragments, 0)
|
|
|
|
|
|
class SkeletonCompressorMediumSuite:
|
|
"""Benchmark compression of a medium fragment set (100 fragments, 10000 tokens)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(100) # 10000 total tokens
|
|
|
|
def time_compress_budget_70pct(self) -> None:
|
|
"""Default-ratio-equivalent compression."""
|
|
self._svc.compress(self._fragments, 7000)
|
|
|
|
def time_compress_budget_50pct(self) -> None:
|
|
"""Moderate compression."""
|
|
self._svc.compress(self._fragments, 5000)
|
|
|
|
|
|
class SkeletonCompressorLargeSuite:
|
|
"""Benchmark compression of a large fragment set (1000 fragments, 100000 tokens)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(1000) # 100000 total tokens
|
|
|
|
def time_compress_budget_50pct(self) -> None:
|
|
"""Moderate compression over 1000 fragments."""
|
|
self._svc.compress(self._fragments, 50000)
|
|
|
|
def time_compress_budget_20pct(self) -> None:
|
|
"""Heavy compression over 1000 fragments."""
|
|
self._svc.compress(self._fragments, 20000)
|