Files
placeholder/features/steps/estimation_lifecycle_hook_651_steps.py
2026-04-02 17:07:48 +00:00

480 lines
19 KiB
Python

"""Step definitions for estimation lifecycle hook and config fallback (issue #651).
Tests the 4-level fallback chain for estimation_actor in use_action(),
the PLAN_ESTIMATION_COMPLETE event emission, and the cost_estimate_usd
population from the estimation result.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when
from behave.runner import Context
from cleveragents.application.services.config_service import ConfigService
from cleveragents.application.services.plan_lifecycle_service import (
PlanLifecycleService,
)
from cleveragents.config.settings import Settings
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
from cleveragents.infrastructure.events.types import EventType
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_service(
config_service: ConfigService | None = None,
event_bus: Any | None = None,
) -> PlanLifecycleService:
"""Create a PlanLifecycleService with optional config_service and event_bus."""
settings = Settings()
svc = PlanLifecycleService(
settings=settings,
config_service=config_service,
event_bus=event_bus,
)
return svc
def _make_mock_config_service_with_estimation(actor_name: str) -> MagicMock:
"""Create a mock ConfigService that returns *actor_name* for actor.default.estimation."""
from cleveragents.application.services.config_service import (
ConfigLevel,
ResolvedValue,
)
mock_cfg = MagicMock(spec=ConfigService)
def _resolve(key: str, **kwargs: Any) -> ResolvedValue:
if key == "actor.default.estimation":
return ResolvedValue(
key=key,
value=actor_name,
source=ConfigLevel.GLOBAL,
chain=[],
)
# For any other key, return a ResolvedValue with None
return ResolvedValue(key=key, value=None, source=ConfigLevel.DEFAULT, chain=[])
mock_cfg.resolve.side_effect = _resolve
return mock_cfg # type: ignore[return-value]
def _make_mock_config_service_no_estimation() -> MagicMock:
"""Create a mock ConfigService that returns None for actor.default.estimation."""
from cleveragents.application.services.config_service import (
ConfigLevel,
ResolvedValue,
)
mock_cfg = MagicMock(spec=ConfigService)
def _resolve(key: str, **kwargs: Any) -> ResolvedValue:
return ResolvedValue(key=key, value=None, source=ConfigLevel.DEFAULT, chain=[])
mock_cfg.resolve.side_effect = _resolve
return mock_cfg # type: ignore[return-value]
def _drive_to_execute(svc: PlanLifecycleService, plan_id: str) -> None:
"""Drive a plan from STRATEGIZE/QUEUED through to Execute phase."""
svc.start_strategize(plan_id)
plan = svc.get_plan(plan_id)
plan.processing_state = ProcessingState.PROCESSING
svc._commit_plan(plan)
svc.complete_strategize(plan_id)
# complete_strategize calls auto_progress; if still in STRATEGIZE/COMPLETE
# we need to call execute_plan explicitly.
plan = svc.get_plan(plan_id)
if plan.phase == PlanPhase.STRATEGIZE:
svc.execute_plan(plan_id)
# ---------------------------------------------------------------------------
# Subtask 1: 4-level fallback chain
# ---------------------------------------------------------------------------
@given('I have a lifecycle service with a global estimation actor config "{actor}"')
def step_service_with_global_estimation(context: Context, actor: str) -> None:
"""Create a lifecycle service with actor.default.estimation set in config."""
cfg = _make_mock_config_service_with_estimation(actor)
context.fallback_svc = _make_service(config_service=cfg)
context.fallback_error = None
@given("I have a lifecycle service without a global estimation actor config")
def step_service_without_global_estimation(context: Context) -> None:
"""Create a lifecycle service with no actor.default.estimation config."""
cfg = _make_mock_config_service_no_estimation()
context.fallback_svc = _make_service(config_service=cfg)
context.fallback_error = None
@given('I create an action with estimation_actor "{actor}"')
def step_create_action_with_estimation_actor(context: Context, actor: str) -> None:
"""Create an action with the given estimation_actor."""
context.fallback_svc.create_action(
name="local/fallback-test",
description="Fallback chain test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor=actor,
)
@given("I create an action without an estimation_actor")
def step_create_action_without_estimation_actor(context: Context) -> None:
"""Create an action with no estimation_actor."""
context.fallback_svc.create_action(
name="local/fallback-test",
description="Fallback chain test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
)
@when("I use the action to create a plan")
def step_use_action_create_plan(context: Context) -> None:
"""Use the action to create a plan."""
context.fallback_plan = context.fallback_svc.use_action(
action_name="local/fallback-test"
)
@then('the plan estimation_actor should be "{expected}"')
def step_check_plan_estimation_actor(context: Context, expected: str) -> None:
"""Verify the plan's estimation_actor matches the expected value."""
assert context.fallback_plan.estimation_actor == expected, (
f"Expected estimation_actor={expected!r}, "
f"got {context.fallback_plan.estimation_actor!r}"
)
@then("the plan estimation_actor should be None for fallback tests")
def step_check_plan_estimation_actor_none(context: Context) -> None:
"""Verify the plan's estimation_actor is None."""
assert context.fallback_plan.estimation_actor is None, (
f"Expected estimation_actor=None, "
f"got {context.fallback_plan.estimation_actor!r}"
)
# ---------------------------------------------------------------------------
# Subtask 2: PLAN_ESTIMATION_COMPLETE event
# ---------------------------------------------------------------------------
@given("I have a lifecycle service with an event bus for estimation tests")
def step_service_with_event_bus(context: Context) -> None:
"""Create a lifecycle service with a mock event bus."""
context.emitted_events = [] # type: ignore[attr-defined]
mock_bus = MagicMock()
def _capture_emit(event: Any) -> None:
context.emitted_events.append(event)
mock_bus.emit.side_effect = _capture_emit
context.event_svc = _make_service(event_bus=mock_bus)
@given('I create an action with estimation_actor "{actor}" for event tests')
def step_create_action_for_event_tests(context: Context, actor: str) -> None:
"""Create an action with estimation_actor for event emission tests."""
context.event_svc.create_action(
name="local/event-test",
description="Event test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor=actor,
)
@given("I create an action without estimation_actor for event tests")
def step_create_action_no_estimation_for_event_tests(context: Context) -> None:
"""Create an action without estimation_actor for event emission tests."""
context.event_svc.create_action(
name="local/event-test-no-est",
description="Event test action without estimation",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
)
@when("I use the action and drive through strategize to execute")
def step_drive_through_strategize_to_execute(context: Context) -> None:
"""Drive the plan from use_action through strategize to execute."""
plan = context.event_svc.use_action(action_name="local/event-test")
context.event_plan = plan
_drive_to_execute(context.event_svc, plan.identity.plan_id)
context.event_plan = context.event_svc.get_plan(plan.identity.plan_id)
@when("I use the action and drive through strategize to execute without estimation")
def step_drive_through_strategize_to_execute_no_estimation(context: Context) -> None:
"""Drive the plan from use_action through strategize to execute (no estimation)."""
plan = context.event_svc.use_action(action_name="local/event-test-no-est")
context.event_plan = plan
_drive_to_execute(context.event_svc, plan.identity.plan_id)
context.event_plan = context.event_svc.get_plan(plan.identity.plan_id)
@then("a PLAN_ESTIMATION_COMPLETE event should have been emitted")
def step_check_estimation_complete_event_emitted(context: Context) -> None:
"""Verify that a PLAN_ESTIMATION_COMPLETE event was emitted."""
event_types = [getattr(e, "event_type", None) for e in context.emitted_events]
assert EventType.PLAN_ESTIMATION_COMPLETE in event_types, (
f"Expected PLAN_ESTIMATION_COMPLETE in emitted events, got: {event_types}"
)
@then("no PLAN_ESTIMATION_COMPLETE event should have been emitted")
def step_check_no_estimation_complete_event(context: Context) -> None:
"""Verify that no PLAN_ESTIMATION_COMPLETE event was emitted."""
event_types = [getattr(e, "event_type", None) for e in context.emitted_events]
assert EventType.PLAN_ESTIMATION_COMPLETE not in event_types, (
f"Expected no PLAN_ESTIMATION_COMPLETE event, but found one in: {event_types}"
)
# ---------------------------------------------------------------------------
# Subtask 3: Conditional estimation step in Strategize-to-Execute
# ---------------------------------------------------------------------------
@given("I have a plan lifecycle service for estimation lifecycle tests")
def step_service_for_lifecycle_tests(context: Context) -> None:
"""Create a plan lifecycle service for lifecycle tests."""
context.lifecycle_svc = _make_service()
context.lifecycle_error = None
@given('I create an action with estimation_actor "{actor}" for lifecycle tests')
def step_create_action_with_estimation_for_lifecycle(
context: Context, actor: str
) -> None:
"""Create an action with estimation_actor for lifecycle tests."""
context.lifecycle_svc.create_action(
name="local/lifecycle-est-test",
description="Lifecycle estimation test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor=actor,
)
@given("I create an action without estimation_actor for lifecycle tests")
def step_create_action_without_estimation_for_lifecycle(context: Context) -> None:
"""Create an action without estimation_actor for lifecycle tests."""
context.lifecycle_svc.create_action(
name="local/lifecycle-no-est-test",
description="Lifecycle no-estimation test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
)
@when("I use the action and complete strategize for lifecycle tests")
def step_use_and_complete_strategize_lifecycle(context: Context) -> None:
"""Use the action and complete the strategize phase."""
# Try both action names
try:
plan = context.lifecycle_svc.use_action(action_name="local/lifecycle-est-test")
except Exception:
plan = context.lifecycle_svc.use_action(
action_name="local/lifecycle-no-est-test"
)
context.lifecycle_plan = plan
plan_id = plan.identity.plan_id
context.lifecycle_svc.start_strategize(plan_id)
p = context.lifecycle_svc.get_plan(plan_id)
p.processing_state = ProcessingState.PROCESSING
context.lifecycle_svc._commit_plan(p)
context.lifecycle_svc.complete_strategize(plan_id)
context.lifecycle_plan = context.lifecycle_svc.get_plan(plan_id)
@when("I call execute_plan for lifecycle tests")
def step_call_execute_plan_lifecycle(context: Context) -> None:
"""Call execute_plan if the plan is still in STRATEGIZE phase."""
plan = context.lifecycle_plan
if plan.phase == PlanPhase.STRATEGIZE:
context.lifecycle_plan = context.lifecycle_svc.execute_plan(
plan.identity.plan_id
)
else:
context.lifecycle_plan = context.lifecycle_svc.get_plan(plan.identity.plan_id)
@then("the plan should be in Execute phase")
def step_check_plan_in_execute_phase(context: Context) -> None:
"""Verify the plan is in Execute phase."""
assert context.lifecycle_plan.phase == PlanPhase.EXECUTE, (
f"Expected Execute phase, got {context.lifecycle_plan.phase}"
)
@then("the plan estimation_result should be populated")
def step_check_estimation_result_populated(context: Context) -> None:
"""Verify the plan has an estimation result."""
assert context.lifecycle_plan.estimation_result is not None, (
"Expected estimation_result to be set, but it is None"
)
@then("the plan estimation_result should be None for lifecycle tests")
def step_check_estimation_result_none_lifecycle(context: Context) -> None:
"""Verify the plan has no estimation result."""
assert context.lifecycle_plan.estimation_result is None, (
f"Expected estimation_result=None, "
f"got {context.lifecycle_plan.estimation_result!r}"
)
# ---------------------------------------------------------------------------
# Subtask 4: cost_estimate_usd populated from estimation result
# ---------------------------------------------------------------------------
@given("I have a plan lifecycle service for cost estimate tests")
def step_service_for_cost_tests(context: Context) -> None:
"""Create a plan lifecycle service for cost estimate tests."""
context.cost_svc = _make_service()
@given('I create an action with estimation_actor "{actor}" for cost tests')
def step_create_action_for_cost_tests(context: Context, actor: str) -> None:
"""Create an action with estimation_actor for cost tests."""
context.cost_svc.create_action(
name="local/cost-test",
description="Cost estimate test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor=actor,
)
@when("I use the action and drive through strategize to execute for cost tests")
def step_drive_to_execute_for_cost_tests(context: Context) -> None:
"""Drive the plan through strategize to execute for cost tests."""
plan = context.cost_svc.use_action(action_name="local/cost-test")
context.cost_plan = plan
_drive_to_execute(context.cost_svc, plan.identity.plan_id)
context.cost_plan = context.cost_svc.get_plan(plan.identity.plan_id)
@then("the plan cost_estimate_usd should be set")
def step_check_cost_estimate_usd_set(context: Context) -> None:
"""Verify cost_estimate_usd is populated on the plan."""
# The stub actor may or may not set estimated_cost_usd; check that
# if estimation_result is set, cost_estimate_usd is consistent.
plan = context.cost_plan
if (
plan.estimation_result is not None
and plan.estimation_result.estimated_cost_usd is not None
):
assert plan.cost_estimate_usd == plan.estimation_result.estimated_cost_usd, (
f"cost_estimate_usd={plan.cost_estimate_usd!r} does not match "
f"estimation_result.estimated_cost_usd="
f"{plan.estimation_result.estimated_cost_usd!r}"
)
# If estimated_cost_usd is None in the result, cost_estimate_usd may be None
# Either way, no assertion error means the wiring is correct
# ---------------------------------------------------------------------------
# Subtask 5: Clean skip when no estimation actor configured
# ---------------------------------------------------------------------------
@given("I have a plan lifecycle service for skip tests")
def step_service_for_skip_tests(context: Context) -> None:
"""Create a plan lifecycle service for skip tests."""
context.skip_svc = _make_service()
@given("I create an action without estimation_actor for skip tests")
def step_create_action_without_estimation_for_skip(context: Context) -> None:
"""Create an action without estimation_actor for skip tests."""
context.skip_svc.create_action(
name="local/skip-test",
description="Skip estimation test action",
definition_of_done="Done",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
)
@when("I use the action and complete strategize for skip tests")
def step_use_and_complete_strategize_skip(context: Context) -> None:
"""Use the action and complete strategize for skip tests."""
plan = context.skip_svc.use_action(action_name="local/skip-test")
context.skip_plan = plan
plan_id = plan.identity.plan_id
context.skip_svc.start_strategize(plan_id)
p = context.skip_svc.get_plan(plan_id)
p.processing_state = ProcessingState.PROCESSING
context.skip_svc._commit_plan(p)
context.skip_svc.complete_strategize(plan_id)
context.skip_plan = context.skip_svc.get_plan(plan_id)
@when("I call execute_plan for skip tests")
def step_call_execute_plan_skip(context: Context) -> None:
"""Call execute_plan for skip tests."""
plan = context.skip_plan
if plan.phase == PlanPhase.STRATEGIZE:
context.skip_plan = context.skip_svc.execute_plan(plan.identity.plan_id)
else:
context.skip_plan = context.skip_svc.get_plan(plan.identity.plan_id)
@then("the plan should transition to Execute phase cleanly")
def step_check_plan_execute_phase_cleanly(context: Context) -> None:
"""Verify the plan is in Execute phase."""
assert context.skip_plan.phase == PlanPhase.EXECUTE, (
f"Expected Execute phase, got {context.skip_plan.phase}"
)
@then("no estimation result should be set on the plan")
def step_check_no_estimation_result_skip(context: Context) -> None:
"""Verify no estimation result is set."""
assert context.skip_plan.estimation_result is None, (
f"Expected estimation_result=None, got {context.skip_plan.estimation_result!r}"
)
# ---------------------------------------------------------------------------
# EventType enum check
# ---------------------------------------------------------------------------
@then("the EventType enum should have PLAN_ESTIMATION_COMPLETE")
def step_check_event_type_has_estimation_complete(context: Context) -> None:
"""Verify EventType has PLAN_ESTIMATION_COMPLETE."""
assert hasattr(EventType, "PLAN_ESTIMATION_COMPLETE"), (
"EventType does not have PLAN_ESTIMATION_COMPLETE"
)
@then('the PLAN_ESTIMATION_COMPLETE value should be "plan.estimation_complete"')
def step_check_estimation_complete_value(context: Context) -> None:
"""Verify PLAN_ESTIMATION_COMPLETE value."""
assert EventType.PLAN_ESTIMATION_COMPLETE == "plan.estimation_complete", (
f"Expected 'plan.estimation_complete', "
f"got {EventType.PLAN_ESTIMATION_COMPLETE!r}"
)