fix(acms): harden hot/warm/cold tier service reliability #11238

Merged
HAL9000 merged 2 commits from pr-fix/9663-hot-warm-cold-tier-reliability into master 2026-05-28 21:31:23 +00:00
@@ -50,22 +50,10 @@ from cleveragents.infrastructure.events.types import EventType
logger = structlog.get_logger(__name__)
# ---------------------------------------------------------------------------
# Default budget when settings are not provided
# Default runtime policy values (sources of truth; settings module has full list)
# ---------------------------------------------------------------------------
_DEFAULT_MAX_TOKENS_HOT = 16000
_DEFAULT_MAX_DECISIONS_WARM = 100
_DEFAULT_MAX_DECISIONS_COLD = 500
# ---------------------------------------------------------------------------
# Default runtime policy values
# ---------------------------------------------------------------------------
_DEFAULT_PROMOTION_THRESHOLD = 5
_DEFAULT_HOT_TTL_HOURS = 24
_DEFAULT_WARM_TTL_HOURS = 24
# Maximum content length kept after cold-tier summarisation
_COLD_SUMMARY_MAX_CHARS = 200
_default_summarisation_max_chars: int = 200
class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
@@ -209,12 +197,12 @@ class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
fragments: list[TieredFragment] = []
for store in (self._hot, self._warm, self._cold):
fragments.extend(store.values())
return fragments
return [f.model_copy() for f in fragments]
def get_hot_fragments(self) -> list[TieredFragment]:
"""Return all fragments currently in the hot tier."""
with self._lock:
return list(self._hot.values())
return [f.model_copy() for f in self._hot.values()]
# ------------------------------------------------------------------
# Actor views
@@ -314,6 +302,20 @@ class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
from_tier=ContextTier.COLD,
to_tier=ContextTier.WARM,
)
# Enforce warm-tier capacity after promotion.
self._enforce_warm_capacity()
# If budget enforcement evicted the just-promoted fragment,
# restore it (touching so it is not immediately re-evicted)
# to prevent silent data loss.
if fragment_id not in self._warm:
promoted = self._touch(promoted)
self._warm[fragment_id] = promoted
self._enforce_warm_capacity()
logger.warning(
"tier.promotion_budget_fallback",
fragment_id=fragment_id,
reason="warm budget exceeded after cold→warm promotion",
)
return promoted
if fragment_id in self._warm:
@@ -336,6 +338,7 @@ class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
update={"tier": ContextTier.WARM},
)
self._warm[fragment_id] = restored
self._enforce_warm_capacity()
logger.warning(
"tier.promotion_budget_fallback",
fragment_id=fragment_id,
@@ -450,6 +453,38 @@ class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
# validate_fragment_scope, store_with_scope_check, get_scoped_metrics)
# are provided by ScopedTierMixin — see scoped_tiers.py.
def _enforce_warm_capacity(self) -> list[str]:
"""Evict LRU warm-tier fragments when fragment capacity is exceeded.
Called automatically by :meth:`promote` after placing a fragment into
the warm tier (either on direct cold→warm promotion or as a fallback
from hot-budget enforcement). Uses count-based eviction rather than
token-based, since the warm tier is an in-memory dict of full-text
fragments.
Returns a list of evicted fragment IDs.
"""
max_warm = self._budget.max_decisions_warm
while len(self._warm) > max_warm and self._warm:
oldest_id = min(
self._warm.keys(),
key=lambda fid: self._warm[fid].last_accessed,
)
del self._warm[oldest_id]
self._emit_tier_event(
EventType.TIER_EVICTED,
oldest_id,
from_tier=ContextTier.WARM,
to_tier=None,
)
logger.info(
"tier.warm_capacity_evicted",
fragment_id=oldest_id,
warm_count=len(self._warm),
capacity=max_warm,
)
return []
# ------------------------------------------------------------------
# Budget
# ------------------------------------------------------------------
@@ -506,8 +541,8 @@ class ContextTierService(TierRuntimeMixin, ScopedTierMixin):
A production implementation would call an LLM summariser.
"""
content = fragment.content
if len(content) > _COLD_SUMMARY_MAX_CHARS:
content = content[:_COLD_SUMMARY_MAX_CHARS] + "..."
if len(content) > _default_summarisation_max_chars:
content = content[:_default_summarisation_max_chars] + "..."
return fragment.model_copy(update={"content": content})