eb46f0ff54
CI / push-validation (push) Successful in 40s
CI / helm (push) Successful in 44s
CI / build (push) Successful in 1m14s
CI / lint (push) Successful in 1m18s
CI / quality (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 1m19s
CI / typecheck (push) Successful in 2m22s
CI / security (push) Successful in 2m23s
CI / benchmark-regression (push) Failing after 41s
CI / integration_tests (push) Successful in 4m46s
CI / unit_tests (push) Failing after 20m13s
CI / benchmark-publish (push) Failing after 28m28s
CI / coverage (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
## Summary This PR fixes issue #10878 where architecture reviews were truncated because the regex pattern for parsing file output would stop at the first ``` encountered in the Markdown report. ## Changes - Change file delimiters from ``` to >>>>>>>/<<<<<<< to avoid Markdown conflicts - Add tier hydration before strategize phase in plan_executor.py - Increase max_tokens to 16384 in llm_actors.py for longer outputs - Increase context_max_tokens_hot from 16000 to 32000 in settings.py - Fix get_hot_view → get_hot_fragments in strategy_actor.py and plan_executor.py - Add opencode to skip directories in context_tier_hydrator.py - Change sandbox output location to plan-output/ directory in plan.py - Add get_context_summary stub method to acms_service.py ## Testing Run architecture review action and verify the output report is complete with all sections. Reviewed-on: #10938
503 lines
16 KiB
Python
503 lines
16 KiB
Python
"""Step definitions for the ACMS Context Tiers feature."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from pydantic import ValidationError
|
|
|
|
from cleveragents.application.container import get_container, reset_container
|
|
from cleveragents.application.services.context_tiers import ContextTierService
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.domain.models.acms.tiers import (
|
|
ActorContextView,
|
|
ActorRole,
|
|
ContextTier,
|
|
ScopedBackendView,
|
|
TierBudget,
|
|
TieredFragment,
|
|
TierMetrics,
|
|
)
|
|
|
|
__all__: list[str] = []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTier enum
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('ContextTier should have values "hot", "warm", "cold"')
|
|
def step_then_context_tier_values(context: Any) -> None:
|
|
assert ContextTier.HOT == "hot"
|
|
assert ContextTier.WARM == "warm"
|
|
assert ContextTier.COLD == "cold"
|
|
assert len(ContextTier) == 3
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ActorRole enum
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('ActorRole should have values "strategist", "executor", "reviewer", "estimator"')
|
|
def step_then_actor_role_values(context: Any) -> None:
|
|
assert ActorRole.STRATEGIST == "strategist"
|
|
assert ActorRole.EXECUTOR == "executor"
|
|
assert ActorRole.REVIEWER == "reviewer"
|
|
assert ActorRole.ESTIMATOR == "estimator"
|
|
assert "estimator" in [r.value for r in ActorRole]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TieredFragment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a TieredFragment with id "{fid}" and content "{content}"')
|
|
def step_given_tiered_fragment(context: Any, fid: str, content: str) -> None:
|
|
context.fragment = TieredFragment(fragment_id=fid, content=content)
|
|
|
|
|
|
@then('the fragment tier should be "{tier}"')
|
|
def step_then_fragment_tier(context: Any, tier: str) -> None:
|
|
assert context.fragment.tier == tier
|
|
|
|
|
|
@then("the fragment token_count should be {count:d}")
|
|
def step_then_fragment_token_count(context: Any, count: int) -> None:
|
|
assert context.fragment.token_count == count
|
|
|
|
|
|
@then("the fragment access_count should be {count:d}")
|
|
def step_then_fragment_access_count(context: Any, count: int) -> None:
|
|
assert context.fragment.access_count == count
|
|
|
|
|
|
@then("creating a TieredFragment with empty fragment_id should raise ValidationError")
|
|
def step_then_empty_fragment_id(context: Any) -> None:
|
|
try:
|
|
TieredFragment(fragment_id="", content="x")
|
|
raise AssertionError("Expected ValidationError")
|
|
except ValidationError:
|
|
pass
|
|
|
|
|
|
@then(
|
|
"creating a TieredFragment with negative token_count should raise ValidationError"
|
|
)
|
|
def step_then_negative_token_count(context: Any) -> None:
|
|
try:
|
|
TieredFragment(fragment_id="f", content="x", token_count=-1)
|
|
raise AssertionError("Expected ValidationError")
|
|
except ValidationError:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TierBudget
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a default TierBudget")
|
|
def step_given_default_tier_budget(context: Any) -> None:
|
|
context.budget = TierBudget()
|
|
|
|
|
|
@then("max_tokens_hot should be {val:d}")
|
|
def step_then_max_tokens_hot(context: Any, val: int) -> None:
|
|
assert context.budget.max_tokens_hot == val
|
|
|
|
|
|
@then("max_decisions_warm should be {val:d}")
|
|
def step_then_max_decisions_warm(context: Any, val: int) -> None:
|
|
assert context.budget.max_decisions_warm == val
|
|
|
|
|
|
@then("max_decisions_cold should be {val:d}")
|
|
def step_then_max_decisions_cold(context: Any, val: int) -> None:
|
|
assert context.budget.max_decisions_cold == val
|
|
|
|
|
|
@then(
|
|
'creating a TierBudget with hot {hot:d}, warm {warm:d}, cold {cold:d} should fail for "{field}"'
|
|
)
|
|
def step_then_tier_budget_rejects(
|
|
context: Any,
|
|
hot: int,
|
|
warm: int,
|
|
cold: int,
|
|
field: str,
|
|
) -> None:
|
|
try:
|
|
TierBudget(
|
|
max_tokens_hot=hot,
|
|
max_decisions_warm=warm,
|
|
max_decisions_cold=cold,
|
|
)
|
|
raise AssertionError("Expected ValidationError")
|
|
except ValidationError as exc:
|
|
error_fields = {
|
|
".".join(str(part) for part in err["loc"]) for err in exc.errors()
|
|
}
|
|
assert field in error_fields, (
|
|
f"Expected validation error for {field}, got {error_fields}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ActorContextView
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('an ActorContextView for "{role}"')
|
|
def step_given_actor_context_view(context: Any, role: str) -> None:
|
|
context.actor_view = ActorContextView(actor_role=ActorRole(role))
|
|
|
|
|
|
@then('effective tiers should be "hot", "warm", "cold"')
|
|
def step_then_all_tiers(context: Any) -> None:
|
|
tiers = context.actor_view.get_effective_tiers()
|
|
assert tiers == [ContextTier.HOT, ContextTier.WARM, ContextTier.COLD]
|
|
|
|
|
|
@then('effective tiers should be "hot", "warm"')
|
|
def step_then_hot_warm(context: Any) -> None:
|
|
tiers = context.actor_view.get_effective_tiers()
|
|
assert tiers == [ContextTier.HOT, ContextTier.WARM]
|
|
|
|
|
|
@then('effective tiers should be "hot"')
|
|
def step_then_hot_only(context: Any) -> None:
|
|
tiers = context.actor_view.get_effective_tiers()
|
|
assert tiers == [ContextTier.HOT]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TierMetrics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a default TierMetrics")
|
|
def step_given_default_metrics(context: Any) -> None:
|
|
context.metrics = TierMetrics()
|
|
|
|
|
|
@then("total_fragments should be {val:d}")
|
|
def step_then_total_fragments(context: Any, val: int) -> None:
|
|
assert context.metrics.total_fragments == val
|
|
|
|
|
|
@then("hot_hit_rate should be {val:g}")
|
|
def step_then_hot_hit_rate(context: Any, val: float) -> None:
|
|
assert context.metrics.hot_hit_rate == val
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ScopedBackendView
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a ScopedBackendView for project "{proj}"')
|
|
def step_given_scoped_view(context: Any, proj: str) -> None:
|
|
context.scoped_view = ScopedBackendView(allowed_projects=frozenset({proj}))
|
|
if not hasattr(context, "view_fragments"):
|
|
context.view_fragments = {}
|
|
|
|
|
|
@given('a TieredFragment with id "{fid}" and no project')
|
|
def step_given_fragment_no_project(context: Any, fid: str) -> None:
|
|
if not hasattr(context, "view_fragments"):
|
|
context.view_fragments = {}
|
|
context.view_fragments[fid] = TieredFragment(
|
|
fragment_id=fid, content="x", project_name=""
|
|
)
|
|
|
|
|
|
@given('a TieredFragment with id "{fid}" in project "{proj}"')
|
|
def step_given_fragment_in_project(context: Any, fid: str, proj: str) -> None:
|
|
if not hasattr(context, "view_fragments"):
|
|
context.view_fragments = {}
|
|
context.view_fragments[fid] = TieredFragment(
|
|
fragment_id=fid, content="x", project_name=proj
|
|
)
|
|
|
|
|
|
@then('fragment "{fid}" should be visible')
|
|
def step_then_fragment_visible(context: Any, fid: str) -> None:
|
|
frag = context.view_fragments[fid]
|
|
assert context.scoped_view.is_visible(frag)
|
|
|
|
|
|
@then('fragment "{fid}" should not be visible')
|
|
def step_then_fragment_not_visible(context: Any, fid: str) -> None:
|
|
frag = context.view_fragments[fid]
|
|
assert not context.scoped_view.is_visible(frag)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: store/get
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a ContextTierService")
|
|
def step_given_tier_service(context: Any) -> None:
|
|
context.tier_service = ContextTierService()
|
|
|
|
|
|
@when('I store a fragment with id "{fid}" content "{content}" in tier "{tier}"')
|
|
def step_when_store_fragment(context: Any, fid: str, content: str, tier: str) -> None:
|
|
frag = TieredFragment(
|
|
fragment_id=fid,
|
|
content=content,
|
|
tier=ContextTier(tier),
|
|
)
|
|
context.tier_service.store(frag)
|
|
|
|
|
|
@when(
|
|
'I store fragment "{fid}" with content "{content}" tier "{tier}" project "{proj}"'
|
|
)
|
|
def step_when_store_fragment_project(
|
|
context: Any, fid: str, content: str, tier: str, proj: str
|
|
) -> None:
|
|
frag = TieredFragment(
|
|
fragment_id=fid,
|
|
content=content,
|
|
tier=ContextTier(tier),
|
|
project_name=proj,
|
|
)
|
|
context.tier_service.store(frag)
|
|
|
|
|
|
@when('I store a long fragment "{fid}" in tier "{tier}"')
|
|
def step_when_store_long_fragment(context: Any, fid: str, tier: str) -> None:
|
|
long_content = "x" * 300 # Longer than _COLD_SUMMARY_MAX_CHARS (200)
|
|
frag = TieredFragment(
|
|
fragment_id=fid,
|
|
content=long_content,
|
|
tier=ContextTier(tier),
|
|
)
|
|
context.tier_service.store(frag)
|
|
|
|
|
|
@then('tier getting "{fid}" should return content "{content}"')
|
|
def step_then_get_content(context: Any, fid: str, content: str) -> None:
|
|
result = context.tier_service.get(fid)
|
|
assert result is not None
|
|
assert result.content == content
|
|
|
|
|
|
@then('tier getting "{fid}" should return None')
|
|
def step_then_get_none(context: Any, fid: str) -> None:
|
|
assert context.tier_service.get(fid) is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: promote/demote
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I promote "{fid}"')
|
|
def step_when_promote(context: Any, fid: str) -> None:
|
|
context.last_promoted = context.tier_service.promote(fid)
|
|
|
|
|
|
@when('I demote "{fid}"')
|
|
def step_when_demote(context: Any, fid: str) -> None:
|
|
context.last_demoted = context.tier_service.demote(fid)
|
|
|
|
|
|
@then('"{fid}" should be in tier "{tier}"')
|
|
def step_then_in_tier(context: Any, fid: str, tier: str) -> None:
|
|
result = context.tier_service.get(fid)
|
|
assert result is not None
|
|
assert result.tier == tier
|
|
|
|
|
|
@then('promoting "{fid}" should return None')
|
|
def step_then_promote_none(context: Any, fid: str) -> None:
|
|
assert context.tier_service.promote(fid) is None
|
|
|
|
|
|
@then('demoting "{fid}" should return None')
|
|
def step_then_demote_none(context: Any, fid: str) -> None:
|
|
assert context.tier_service.demote(fid) is None
|
|
|
|
|
|
@then('"{fid}" content should be summarized')
|
|
def step_then_content_summarized(context: Any, fid: str) -> None:
|
|
result = context.tier_service.get(fid)
|
|
assert result is not None
|
|
# Summarised content ends with "..." if original was long
|
|
assert result.content.endswith("...")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: LRU eviction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I evict {count:d} LRU fragments from "{tier}"')
|
|
def step_when_evict(context: Any, count: int, tier: str) -> None:
|
|
context.evicted = context.tier_service.evict_lru(ContextTier(tier), count)
|
|
|
|
|
|
@then('"{tier}" tier should have {count:d} fragment')
|
|
def step_then_tier_count_singular(context: Any, tier: str, count: int) -> None:
|
|
metrics = context.tier_service.get_metrics()
|
|
if tier == "hot":
|
|
assert metrics.hot_count == count
|
|
elif tier == "warm":
|
|
assert metrics.warm_count == count
|
|
else:
|
|
assert metrics.cold_count == count
|
|
|
|
|
|
@then('evicting {count:d} from "{tier}" should raise ValueError')
|
|
def step_then_evict_raises(context: Any, count: int, tier: str) -> None:
|
|
try:
|
|
context.tier_service.evict_lru(ContextTier(tier), count)
|
|
raise AssertionError("Expected ValueError")
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: actor views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('strategist for project "{proj}" should see {count:d} fragments')
|
|
def step_then_strategist_sees(context: Any, proj: str, count: int) -> None:
|
|
frags = context.tier_service.get_for_actor(
|
|
ActorRole.STRATEGIST, project_names=[proj]
|
|
)
|
|
assert len(frags) == count
|
|
|
|
|
|
@then('executor for project "{proj}" should see {count:d} fragments')
|
|
def step_then_executor_sees(context: Any, proj: str, count: int) -> None:
|
|
frags = context.tier_service.get_for_actor(ActorRole.EXECUTOR, project_names=[proj])
|
|
assert len(frags) == count
|
|
|
|
|
|
@then('reviewer for project "{proj}" should see {count:d} fragment')
|
|
def step_then_reviewer_sees(context: Any, proj: str, count: int) -> None:
|
|
frags = context.tier_service.get_for_actor(ActorRole.REVIEWER, project_names=[proj])
|
|
assert len(frags) == count
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: scoped view
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('scoped view for "{proj}" should have {count:d} fragment')
|
|
def step_then_scoped_count(context: Any, proj: str, count: int) -> None:
|
|
frags = context.tier_service.get_scoped_view([proj])
|
|
assert len(frags) == count
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ContextTierService: metrics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I get fragment "{fid}"')
|
|
def step_when_get_fragment(context: Any, fid: str) -> None:
|
|
context.tier_service.get(fid)
|
|
|
|
|
|
@then("hot_hit_count should be {val:d}")
|
|
def step_then_hot_hit_count(context: Any, val: int) -> None:
|
|
metrics = context.tier_service.get_metrics()
|
|
assert metrics.hot_hit_count == val
|
|
|
|
|
|
@then("hot_miss_count should be {val:d}")
|
|
def step_then_hot_miss_count(context: Any, val: int) -> None:
|
|
metrics = context.tier_service.get_metrics()
|
|
assert metrics.hot_miss_count == val
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DI Container
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("the DI container is reset")
|
|
def step_given_di_reset(context: Any) -> None:
|
|
reset_container()
|
|
context.container = get_container()
|
|
|
|
|
|
@then("the container should provide a context_tier_service")
|
|
def step_then_container_has_tier_service(context: Any) -> None:
|
|
context.resolved_tier_svc = context.container.context_tier_service()
|
|
assert context.resolved_tier_svc is not None
|
|
|
|
|
|
@then("the context_tier_service should be a ContextTierService")
|
|
def step_then_tier_svc_type(context: Any) -> None:
|
|
assert isinstance(context.resolved_tier_svc, ContextTierService)
|
|
|
|
|
|
@then("resolving context_tier_service twice should return the same instance")
|
|
def step_then_tier_svc_singleton(context: Any) -> None:
|
|
a = context.container.context_tier_service()
|
|
b = context.container.context_tier_service()
|
|
assert a is b
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Settings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("settings should have context_max_tokens_hot")
|
|
def step_then_settings_hot(context: Any) -> None:
|
|
s = Settings()
|
|
assert hasattr(s, "context_max_tokens_hot")
|
|
assert s.context_max_tokens_hot == 32000 # Doubled from 16000 to 32000 (M5 fix)
|
|
|
|
|
|
@then("settings should have context_max_decisions_warm")
|
|
def step_then_settings_warm(context: Any) -> None:
|
|
s = Settings()
|
|
assert hasattr(s, "context_max_decisions_warm")
|
|
assert s.context_max_decisions_warm == 100
|
|
|
|
|
|
@then("settings should have context_max_decisions_cold")
|
|
def step_then_settings_cold(context: Any) -> None:
|
|
s = Settings()
|
|
assert hasattr(s, "context_max_decisions_cold")
|
|
assert s.context_max_decisions_cold == 500
|
|
|
|
|
|
@when("I create settings with context tier values {hot:d}, {warm:d}, {cold:d}")
|
|
def step_when_create_settings_with_values(
|
|
context: Any, hot: int, warm: int, cold: int
|
|
) -> None:
|
|
try:
|
|
context.settings_result = Settings(
|
|
context_max_tokens_hot=hot,
|
|
context_max_decisions_warm=warm,
|
|
context_max_decisions_cold=cold,
|
|
)
|
|
context.settings_error = None
|
|
except ValidationError as exc:
|
|
context.settings_result = None
|
|
context.settings_error = exc
|
|
|
|
|
|
@then('settings validation should fail for "{field}"')
|
|
def step_then_settings_validation_failure(context: Any, field: str) -> None:
|
|
assert context.settings_error is not None, "Expected validation to fail"
|
|
errors = context.settings_error.errors()
|
|
assert any(field in err.get("loc", ()) for err in errors), (
|
|
f"Expected validation error for {field}, got: {errors}"
|
|
)
|