brent.edwards
eb46f0ff54
CI / push-validation (push) Successful in 40s
CI / helm (push) Successful in 44s
CI / build (push) Successful in 1m14s
CI / lint (push) Successful in 1m18s
CI / quality (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 1m19s
CI / typecheck (push) Successful in 2m22s
CI / security (push) Successful in 2m23s
CI / benchmark-regression (push) Failing after 41s
CI / integration_tests (push) Successful in 4m46s
CI / unit_tests (push) Failing after 20m13s
CI / benchmark-publish (push) Failing after 28m28s
CI / coverage (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
## Summary This PR fixes issue #10878 where architecture reviews were truncated because the regex pattern for parsing file output would stop at the first ``` encountered in the Markdown report. ## Changes - Change file delimiters from ``` to >>>>>>>/<<<<<<< to avoid Markdown conflicts - Add tier hydration before strategize phase in plan_executor.py - Increase max_tokens to 16384 in llm_actors.py for longer outputs - Increase context_max_tokens_hot from 16000 to 32000 in settings.py - Fix get_hot_view → get_hot_fragments in strategy_actor.py and plan_executor.py - Add opencode to skip directories in context_tier_hydrator.py - Change sandbox output location to plan-output/ directory in plan.py - Add get_context_summary stub method to acms_service.py ## Testing Run architecture review action and verify the output report is complete with all sections. Reviewed-on: #10938
519 lines
19 KiB
Python
519 lines
19 KiB
Python
"""Step definitions for invariant_reconciliation_autowire.feature.
|
|
|
|
Tests the auto-invocation of the Invariant Reconciliation Actor
|
|
during PlanLifecycleService phase transitions: start_strategize,
|
|
execute_plan, and apply_plan, as well as post-correction
|
|
reconciliation via event subscription.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
from behave import given, then, when # type: ignore[import-untyped]
|
|
from behave.runner import Context # type: ignore[import-untyped]
|
|
|
|
from cleveragents.application.services.decision_service import DecisionService
|
|
from cleveragents.application.services.invariant_service import InvariantService
|
|
from cleveragents.application.services.plan_lifecycle_service import (
|
|
PlanLifecycleService,
|
|
ReconciliationBlockedError,
|
|
)
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.domain.models.core.invariant import InvariantScope
|
|
from cleveragents.domain.models.core.plan import (
|
|
PlanPhase,
|
|
ProcessingState,
|
|
)
|
|
from cleveragents.infrastructure.events.models import DomainEvent
|
|
from cleveragents.infrastructure.events.types import EventType
|
|
|
|
# ================================================================
|
|
# Mock Event Bus
|
|
# ================================================================
|
|
|
|
|
|
class _MockEventBus:
|
|
"""Minimal event bus that records emitted events and dispatches subscribers."""
|
|
|
|
def __init__(self) -> None:
|
|
self.events: list[DomainEvent] = []
|
|
self._subscribers: dict[EventType, list[object]] = {}
|
|
|
|
def emit(self, event: DomainEvent) -> None:
|
|
self.events.append(event)
|
|
handlers = self._subscribers.get(event.event_type, [])
|
|
for handler in handlers:
|
|
handler(event) # type: ignore[operator]
|
|
|
|
def subscribe(self, event_type: EventType, handler: object) -> None:
|
|
self._subscribers.setdefault(event_type, []).append(handler)
|
|
|
|
|
|
# ================================================================
|
|
# Helpers
|
|
# ================================================================
|
|
|
|
|
|
def _create_service(
|
|
context: Context,
|
|
invariant_service: InvariantService | None = None,
|
|
event_bus: _MockEventBus | None = None,
|
|
) -> PlanLifecycleService:
|
|
"""Create a PlanLifecycleService with invariant reconciliation wired."""
|
|
Settings._instance = None
|
|
settings = Settings()
|
|
inv_svc = invariant_service or InvariantService()
|
|
dec_svc = DecisionService()
|
|
bus = event_bus or _MockEventBus()
|
|
service = PlanLifecycleService(
|
|
settings=settings,
|
|
decision_service=dec_svc,
|
|
event_bus=bus,
|
|
invariant_service=inv_svc,
|
|
)
|
|
context.service = service
|
|
context.invariant_service = inv_svc
|
|
context.decision_service = dec_svc
|
|
context.event_bus = bus
|
|
context.error = None
|
|
context.reconciliation_invoked = False
|
|
return service
|
|
|
|
|
|
def _create_plan_at_phase(
|
|
context: Context,
|
|
phase: PlanPhase,
|
|
state: ProcessingState,
|
|
invariant_actor: str | None,
|
|
) -> str:
|
|
"""Create a plan and advance it to the given phase/state."""
|
|
svc: PlanLifecycleService = context.service
|
|
|
|
# Create action
|
|
action = svc.create_action(
|
|
name="test-action-autowire",
|
|
description="Test action for reconciliation autowire",
|
|
definition_of_done="All tests pass",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
invariant_actor=invariant_actor,
|
|
)
|
|
|
|
# Use action to create plan (in Strategize/QUEUED)
|
|
action_name = str(action.namespaced_name)
|
|
plan = svc.use_action(action_name=action_name)
|
|
plan_id = plan.identity.plan_id
|
|
|
|
# Advance plan to desired phase/state by directly setting fields
|
|
if phase == PlanPhase.STRATEGIZE and state == ProcessingState.QUEUED:
|
|
# Already there after use_action
|
|
pass
|
|
elif phase == PlanPhase.STRATEGIZE and state == ProcessingState.COMPLETE:
|
|
plan.processing_state = ProcessingState.PROCESSING
|
|
plan.processing_state = ProcessingState.COMPLETE
|
|
svc.commit_plan(plan)
|
|
elif phase == PlanPhase.STRATEGIZE and state == ProcessingState.PROCESSING:
|
|
plan.processing_state = ProcessingState.PROCESSING
|
|
svc.commit_plan(plan)
|
|
elif phase == PlanPhase.EXECUTE and state == ProcessingState.COMPLETE:
|
|
plan.processing_state = ProcessingState.COMPLETE
|
|
svc.commit_plan(plan)
|
|
# Transition to Execute
|
|
plan.processing_state = ProcessingState.QUEUED
|
|
plan.phase = PlanPhase.EXECUTE
|
|
svc.commit_plan(plan)
|
|
# Advance to complete
|
|
plan.processing_state = ProcessingState.PROCESSING
|
|
plan.processing_state = ProcessingState.COMPLETE
|
|
svc.commit_plan(plan)
|
|
|
|
context.plan = svc.get_plan(plan_id)
|
|
context.plan_id = plan_id
|
|
return plan_id
|
|
|
|
|
|
# ================================================================
|
|
# Background
|
|
# ================================================================
|
|
|
|
|
|
@given("a fresh PlanLifecycleService with invariant reconciliation wired")
|
|
def step_fresh_service(context: Context) -> None:
|
|
"""Create a PlanLifecycleService with invariant reconciliation configured."""
|
|
_create_service(context)
|
|
|
|
|
|
# ================================================================
|
|
# Plan creation at various phases
|
|
# ================================================================
|
|
|
|
|
|
@given('a plan in Strategize/QUEUED with invariant_actor "{actor}"')
|
|
def step_plan_strategize_queued_with_actor(context: Context, actor: str) -> None:
|
|
"""Create a plan in Strategize/QUEUED with the given invariant_actor."""
|
|
_create_plan_at_phase(context, PlanPhase.STRATEGIZE, ProcessingState.QUEUED, actor)
|
|
|
|
|
|
@given("a plan in Strategize/QUEUED with invariant_actor unset")
|
|
def step_plan_strategize_queued_no_actor(context: Context) -> None:
|
|
"""Create a plan in Strategize/QUEUED with no invariant_actor."""
|
|
_create_plan_at_phase(context, PlanPhase.STRATEGIZE, ProcessingState.QUEUED, None)
|
|
|
|
|
|
@given('a plan in Strategize/COMPLETE with invariant_actor "{actor}"')
|
|
def step_plan_strategize_complete_with_actor(context: Context, actor: str) -> None:
|
|
"""Create a plan in Strategize/COMPLETE."""
|
|
_create_plan_at_phase(
|
|
context, PlanPhase.STRATEGIZE, ProcessingState.COMPLETE, actor
|
|
)
|
|
|
|
|
|
@given('a plan in Execute/COMPLETE with invariant_actor "{actor}"')
|
|
def step_plan_execute_complete_with_actor(context: Context, actor: str) -> None:
|
|
"""Create a plan in Execute/COMPLETE."""
|
|
_create_plan_at_phase(context, PlanPhase.EXECUTE, ProcessingState.COMPLETE, actor)
|
|
|
|
|
|
@given('a plan in Strategize/PROCESSING with invariant_actor "{actor}"')
|
|
def step_plan_strategize_processing_with_actor(context: Context, actor: str) -> None:
|
|
"""Create a plan in Strategize/PROCESSING."""
|
|
_create_plan_at_phase(
|
|
context, PlanPhase.STRATEGIZE, ProcessingState.PROCESSING, actor
|
|
)
|
|
|
|
|
|
# ================================================================
|
|
# Invariant setup
|
|
# ================================================================
|
|
|
|
|
|
@given('a global invariant "{text}" is registered')
|
|
def step_register_global_invariant(context: Context, text: str) -> None:
|
|
"""Register a global invariant in the invariant service."""
|
|
context.invariant_service.add_invariant(
|
|
text=text,
|
|
scope=InvariantScope.GLOBAL,
|
|
source_name="system",
|
|
)
|
|
|
|
|
|
@given('a plan invariant "{text}" is registered for the plan')
|
|
def step_register_plan_invariant(context: Context, text: str) -> None:
|
|
"""Register a plan-scoped invariant for the current plan."""
|
|
context.invariant_service.add_invariant(
|
|
text=text,
|
|
scope=InvariantScope.PLAN,
|
|
source_name=context.plan_id,
|
|
)
|
|
|
|
|
|
@given("the invariant service is configured to raise an error")
|
|
def step_invariant_service_raises(context: Context) -> None:
|
|
"""Configure the invariant service to raise on list_invariants."""
|
|
|
|
def _raise_error(*args: object, **kwargs: object) -> list[object]:
|
|
raise RuntimeError("Simulated invariant service failure")
|
|
|
|
context.invariant_service.list_invariants = _raise_error # type: ignore[assignment]
|
|
|
|
|
|
# ================================================================
|
|
# Actions: successful calls
|
|
# ================================================================
|
|
|
|
|
|
@when("I call start_strategize on the plan")
|
|
def step_call_start_strategize(context: Context) -> None:
|
|
"""Call start_strategize and track reconciliation invocation."""
|
|
svc: PlanLifecycleService = context.service
|
|
with patch(
|
|
"cleveragents.actor.reconciliation.InvariantReconciliationActor.run",
|
|
wraps=None,
|
|
) as mock_run:
|
|
# Set up the mock to return a valid result
|
|
from cleveragents.actor.reconciliation import ReconciliationResult
|
|
from cleveragents.domain.models.core.invariant import InvariantSet
|
|
|
|
mock_run.return_value = ReconciliationResult(
|
|
reconciled_set=InvariantSet(invariants=[]),
|
|
conflicts=[],
|
|
enforced_decision_ids=[],
|
|
)
|
|
try:
|
|
context.plan = svc.start_strategize(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
context.reconciliation_invoked = mock_run.called
|
|
|
|
|
|
@when("I call start_strategize on the plan without mocking")
|
|
def step_call_start_strategize_real(context: Context) -> None:
|
|
"""Call start_strategize with real reconciliation (no mocking)."""
|
|
svc: PlanLifecycleService = context.service
|
|
try:
|
|
context.plan = svc.start_strategize(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
context.reconciliation_invoked = True
|
|
|
|
|
|
@when("I call execute_plan on the plan")
|
|
def step_call_execute_plan(context: Context) -> None:
|
|
"""Call execute_plan and track reconciliation invocation."""
|
|
svc: PlanLifecycleService = context.service
|
|
with patch(
|
|
"cleveragents.actor.reconciliation.InvariantReconciliationActor.run",
|
|
wraps=None,
|
|
) as mock_run:
|
|
from cleveragents.actor.reconciliation import ReconciliationResult
|
|
from cleveragents.domain.models.core.invariant import InvariantSet
|
|
|
|
mock_run.return_value = ReconciliationResult(
|
|
reconciled_set=InvariantSet(invariants=[]),
|
|
conflicts=[],
|
|
enforced_decision_ids=[],
|
|
)
|
|
try:
|
|
context.plan = svc.execute_plan(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
context.reconciliation_invoked = mock_run.called
|
|
|
|
|
|
@when("I call apply_plan on the plan")
|
|
def step_call_apply_plan(context: Context) -> None:
|
|
"""Call apply_plan and track reconciliation invocation."""
|
|
svc: PlanLifecycleService = context.service
|
|
with patch(
|
|
"cleveragents.actor.reconciliation.InvariantReconciliationActor.run",
|
|
wraps=None,
|
|
) as mock_run:
|
|
from cleveragents.actor.reconciliation import ReconciliationResult
|
|
from cleveragents.domain.models.core.invariant import InvariantSet
|
|
|
|
mock_run.return_value = ReconciliationResult(
|
|
reconciled_set=InvariantSet(invariants=[]),
|
|
conflicts=[],
|
|
enforced_decision_ids=[],
|
|
)
|
|
try:
|
|
context.plan = svc.apply_plan(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
context.reconciliation_invoked = mock_run.called
|
|
|
|
|
|
# ================================================================
|
|
# Actions: failure scenarios
|
|
# ================================================================
|
|
|
|
|
|
@when("I attempt to call start_strategize on the plan")
|
|
def step_attempt_start_strategize(context: Context) -> None:
|
|
"""Call start_strategize expecting it to fail."""
|
|
svc: PlanLifecycleService = context.service
|
|
try:
|
|
context.plan = svc.start_strategize(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I attempt to call execute_plan on the plan")
|
|
def step_attempt_execute_plan(context: Context) -> None:
|
|
"""Call execute_plan expecting it to fail."""
|
|
svc: PlanLifecycleService = context.service
|
|
try:
|
|
context.plan = svc.execute_plan(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
|
|
|
|
@when("I attempt to call apply_plan on the plan")
|
|
def step_attempt_apply_plan(context: Context) -> None:
|
|
"""Call apply_plan expecting it to fail."""
|
|
svc: PlanLifecycleService = context.service
|
|
try:
|
|
context.plan = svc.apply_plan(context.plan_id)
|
|
except Exception as exc:
|
|
context.error = exc
|
|
|
|
|
|
# ================================================================
|
|
# Post-correction event
|
|
# ================================================================
|
|
|
|
|
|
@when("a CORRECTION_APPLIED event is emitted for the plan")
|
|
def step_emit_correction_applied(context: Context) -> None:
|
|
"""Simulate a CORRECTION_APPLIED event to trigger reconciliation."""
|
|
event = DomainEvent(
|
|
event_type=EventType.CORRECTION_APPLIED,
|
|
plan_id=context.plan_id,
|
|
details={
|
|
"correction_id": "test-correction-id",
|
|
"target_decision_id": "test-decision-id",
|
|
"mode": "revert",
|
|
"guidance": "test guidance",
|
|
},
|
|
)
|
|
context.event_bus.emit(event)
|
|
|
|
|
|
# ================================================================
|
|
# Assertions: plan state
|
|
# ================================================================
|
|
|
|
|
|
@then("the plan should transition to Strategize/PROCESSING")
|
|
def step_plan_strategize_processing(context: Context) -> None:
|
|
"""Verify plan is in Strategize/PROCESSING."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.STRATEGIZE, f"Expected STRATEGIZE, got {plan.phase}"
|
|
assert plan.state == ProcessingState.PROCESSING, (
|
|
f"Expected PROCESSING, got {plan.state}"
|
|
)
|
|
|
|
|
|
@then("the plan should transition to Execute/QUEUED")
|
|
def step_plan_execute_queued(context: Context) -> None:
|
|
"""Verify plan is in Execute/QUEUED."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.EXECUTE, f"Expected EXECUTE, got {plan.phase}"
|
|
assert plan.state == ProcessingState.QUEUED, f"Expected QUEUED, got {plan.state}"
|
|
|
|
|
|
@then("the plan should transition to Apply/QUEUED")
|
|
def step_plan_apply_queued(context: Context) -> None:
|
|
"""Verify plan is in Apply/QUEUED."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.APPLY, f"Expected APPLY, got {plan.phase}"
|
|
assert plan.state == ProcessingState.QUEUED, f"Expected QUEUED, got {plan.state}"
|
|
|
|
|
|
@then("the plan should remain in Strategize/QUEUED")
|
|
def step_plan_remain_strategize_queued(context: Context) -> None:
|
|
"""Verify plan is still in Strategize/QUEUED (transition blocked)."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.STRATEGIZE, f"Expected STRATEGIZE, got {plan.phase}"
|
|
assert plan.state == ProcessingState.QUEUED, f"Expected QUEUED, got {plan.state}"
|
|
|
|
|
|
@then("the plan should remain in Strategize/COMPLETE")
|
|
def step_plan_remain_strategize_complete(context: Context) -> None:
|
|
"""Verify plan is still in Strategize/COMPLETE (transition blocked)."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.STRATEGIZE, f"Expected STRATEGIZE, got {plan.phase}"
|
|
assert plan.state == ProcessingState.COMPLETE, (
|
|
f"Expected COMPLETE, got {plan.state}"
|
|
)
|
|
|
|
|
|
@then("the plan should remain in Execute/COMPLETE")
|
|
def step_plan_remain_execute_complete(context: Context) -> None:
|
|
"""Verify plan is still in Execute/COMPLETE (transition blocked)."""
|
|
plan = context.service.get_plan(context.plan_id)
|
|
assert plan.phase == PlanPhase.EXECUTE, f"Expected EXECUTE, got {plan.phase}"
|
|
assert plan.state == ProcessingState.COMPLETE, (
|
|
f"Expected COMPLETE, got {plan.state}"
|
|
)
|
|
|
|
|
|
# ================================================================
|
|
# Assertions: reconciliation invocation
|
|
# ================================================================
|
|
|
|
|
|
@then("invariant reconciliation should have been invoked")
|
|
def step_reconciliation_invoked(context: Context) -> None:
|
|
"""Verify that the reconciliation actor was invoked."""
|
|
assert context.reconciliation_invoked, "Expected reconciliation to be invoked"
|
|
|
|
|
|
@then("invariant reconciliation should not have been invoked")
|
|
def step_reconciliation_not_invoked(context: Context) -> None:
|
|
"""Verify that the reconciliation actor was NOT invoked."""
|
|
assert not context.reconciliation_invoked, (
|
|
"Expected reconciliation NOT to be invoked"
|
|
)
|
|
|
|
|
|
@then("invariant reconciliation should have been invoked for the plan")
|
|
def step_reconciliation_invoked_for_plan(context: Context) -> None:
|
|
"""Verify reconciliation ran for the plan (via event handler)."""
|
|
# Check that INVARIANT_RECONCILED or INVARIANT_VIOLATED event was emitted
|
|
# after the CORRECTION_APPLIED event
|
|
events = context.event_bus.events
|
|
reconciled_events = [
|
|
e
|
|
for e in events
|
|
if e.event_type
|
|
in (EventType.INVARIANT_RECONCILED, EventType.INVARIANT_VIOLATED)
|
|
and e.plan_id == context.plan_id
|
|
]
|
|
assert len(reconciled_events) > 0, (
|
|
"Expected INVARIANT_RECONCILED or INVARIANT_VIOLATED event after correction"
|
|
)
|
|
|
|
|
|
# ================================================================
|
|
# Assertions: events
|
|
# ================================================================
|
|
|
|
|
|
@then("an INVARIANT_RECONCILED event should have been emitted")
|
|
def step_invariant_reconciled_event(context: Context) -> None:
|
|
"""Verify an INVARIANT_RECONCILED event was emitted."""
|
|
events = context.event_bus.events
|
|
reconciled = [e for e in events if e.event_type == EventType.INVARIANT_RECONCILED]
|
|
assert len(reconciled) >= 1, (
|
|
f"Expected INVARIANT_RECONCILED event, got events: "
|
|
f"{[e.event_type for e in events]}"
|
|
)
|
|
|
|
|
|
@then("an INVARIANT_VIOLATED event should have been emitted")
|
|
def step_invariant_violated_event(context: Context) -> None:
|
|
"""Verify an INVARIANT_VIOLATED event was emitted."""
|
|
events = context.event_bus.events
|
|
violated = [e for e in events if e.event_type == EventType.INVARIANT_VIOLATED]
|
|
assert len(violated) >= 1, (
|
|
f"Expected INVARIANT_VIOLATED event, got events: "
|
|
f"{[e.event_type for e in events]}"
|
|
)
|
|
|
|
|
|
# ================================================================
|
|
# Assertions: errors
|
|
# ================================================================
|
|
|
|
|
|
@then("a ReconciliationBlockedError should be raised")
|
|
def step_reconciliation_blocked_error(context: Context) -> None:
|
|
"""Verify that a ReconciliationBlockedError was raised."""
|
|
assert context.error is not None, "Expected an error to be raised"
|
|
assert isinstance(context.error, ReconciliationBlockedError), (
|
|
f"Expected ReconciliationBlockedError, got {type(context.error).__name__}: "
|
|
f"{context.error}"
|
|
)
|
|
|
|
|
|
# ================================================================
|
|
# Assertions: decisions
|
|
# ================================================================
|
|
|
|
|
|
@then("{count:d} invariant_enforced decisions should be recorded for the plan")
|
|
def step_invariant_enforced_decisions_count(context: Context, count: int) -> None:
|
|
"""Verify the number of invariant_enforced decisions recorded."""
|
|
from cleveragents.domain.models.core.decision import DecisionType
|
|
|
|
dec_svc: DecisionService = context.decision_service
|
|
decisions = dec_svc.list_decisions(plan_id=context.plan_id)
|
|
enforced = [
|
|
d for d in decisions if d.decision_type == DecisionType.INVARIANT_ENFORCED
|
|
]
|
|
assert len(enforced) == count, (
|
|
f"Expected {count} invariant_enforced decisions, got {len(enforced)}"
|
|
)
|