freemo
fae438a7a7
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 33s
CI / typecheck (pull_request) Successful in 35s
CI / unit_tests (pull_request) Successful in 2m22s
CI / integration_tests (pull_request) Successful in 3m0s
CI / docker (pull_request) Successful in 39s
CI / coverage (pull_request) Successful in 4m21s
CI / lint (push) Successful in 14s
CI / quality (push) Successful in 16s
CI / build (push) Successful in 16s
CI / security (push) Successful in 30s
CI / typecheck (push) Successful in 36s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 3m38s
CI / docker (push) Successful in 39s
CI / integration_tests (push) Successful in 4m25s
CI / coverage (push) Successful in 4m19s
CI / benchmark-publish (push) Successful in 15m57s
CI / benchmark-regression (pull_request) Successful in 28m50s
Core domain types (FragmentProvenance, ContextFragment, ContextBudget, ContextPayload) now extend their CRP counterparts via Pydantic v2 inheritance, ensuring isinstance compatibility across the model hierarchy. Key changes: - CRP base types made frozen=True (no consumer mutates them) - CRP AssembledContext fields changed from list to tuple (frozen consistency) - Core types extend CRP bases: FragmentProvenance(CRPFragmentProvenance), ContextFragment(CRPContextFragment), ContextBudget(CRPContextBudget), ContextPayload(CRPAssembledContext) - Removed duplicate ContextFragment dataclass from skeleton_compressor - Updated project_context.py to pass tuples to frozen AssembledContext - Added Behave tests (10 scenarios), Robot integration tests (3 cases), and ASV benchmarks for the unified hierarchy - Updated Known Limitations table in docs/reference/acms.md ISSUES CLOSED: #569
108 lines
3.4 KiB
Python
108 lines
3.4 KiB
Python
"""ASV benchmarks for skeleton compressor overhead.
|
|
|
|
Measures the time to compress context fragments at various ratios
|
|
and fragment counts. The benchmark covers the hot path that runs
|
|
during subplan context inheritance.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.application.services.skeleton_compressor import ( # noqa: E402
|
|
SkeletonCompressorService,
|
|
)
|
|
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
|
ContextFragment,
|
|
FragmentProvenance,
|
|
)
|
|
|
|
# Default provenance for skeleton benchmark fragments.
|
|
_SKEL_PROV = FragmentProvenance(resource_uri="bench-skeleton://default")
|
|
|
|
|
|
def _build_fragments(count: int, tokens_each: int = 100) -> list[ContextFragment]:
|
|
"""Generate *count* fragments for benchmarking."""
|
|
return [
|
|
ContextFragment(
|
|
fragment_id=f"bench-{i:05d}",
|
|
uko_node=f"bench-skeleton://file/{i}",
|
|
content="x" * tokens_each,
|
|
token_count=tokens_each,
|
|
relevance_score=round(1.0 - (i / max(count, 1)), 4),
|
|
provenance=_SKEL_PROV,
|
|
metadata=(
|
|
{"source_decision_id": f"01HX{'B' * 22}{i:02d}"} if i < 99 else {}
|
|
),
|
|
)
|
|
for i in range(count)
|
|
]
|
|
|
|
|
|
class SkeletonCompressorSmallSuite:
|
|
"""Benchmark compression of a small fragment set (10 fragments)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(10)
|
|
|
|
def time_compress_ratio_0(self) -> None:
|
|
"""No compression (pass-through)."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.0)
|
|
|
|
def time_compress_ratio_03(self) -> None:
|
|
"""Default ratio compression."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.3)
|
|
|
|
def time_compress_ratio_05(self) -> None:
|
|
"""Moderate compression."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.5)
|
|
|
|
def time_compress_ratio_1(self) -> None:
|
|
"""Maximum compression."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=1.0)
|
|
|
|
|
|
class SkeletonCompressorMediumSuite:
|
|
"""Benchmark compression of a medium fragment set (100 fragments)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(100)
|
|
|
|
def time_compress_ratio_03(self) -> None:
|
|
"""Default ratio compression."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.3)
|
|
|
|
def time_compress_ratio_05(self) -> None:
|
|
"""Moderate compression."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.5)
|
|
|
|
|
|
class SkeletonCompressorLargeSuite:
|
|
"""Benchmark compression of a large fragment set (1000 fragments)."""
|
|
|
|
def setup(self) -> None:
|
|
self._svc = SkeletonCompressorService()
|
|
self._fragments = _build_fragments(1000)
|
|
|
|
def time_compress_ratio_05(self) -> None:
|
|
"""Moderate compression over 1000 fragments."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.5)
|
|
|
|
def time_compress_ratio_08(self) -> None:
|
|
"""Heavy compression over 1000 fragments."""
|
|
self._svc.compress(self._fragments, skeleton_ratio=0.8)
|