forked from HAL9000/cleveragents-core
0989f58dc8
Co-authored-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me> Co-committed-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me>
480 lines
19 KiB
Python
480 lines
19 KiB
Python
"""Step definitions for estimation lifecycle hook and config fallback (issue #651).
|
|
|
|
Tests the 4-level fallback chain for estimation_actor in use_action(),
|
|
the PLAN_ESTIMATION_COMPLETE event emission, and the cost_estimate_usd
|
|
population from the estimation result.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.application.services.config_service import ConfigService
|
|
from cleveragents.application.services.plan_lifecycle_service import (
|
|
PlanLifecycleService,
|
|
)
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
|
|
from cleveragents.infrastructure.events.types import EventType
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_service(
|
|
config_service: ConfigService | None = None,
|
|
event_bus: Any | None = None,
|
|
) -> PlanLifecycleService:
|
|
"""Create a PlanLifecycleService with optional config_service and event_bus."""
|
|
settings = Settings()
|
|
svc = PlanLifecycleService(
|
|
settings=settings,
|
|
config_service=config_service,
|
|
event_bus=event_bus,
|
|
)
|
|
return svc
|
|
|
|
|
|
def _make_mock_config_service_with_estimation(actor_name: str) -> MagicMock:
|
|
"""Create a mock ConfigService that returns *actor_name* for actor.default.estimation."""
|
|
from cleveragents.application.services.config_service import (
|
|
ConfigLevel,
|
|
ResolvedValue,
|
|
)
|
|
|
|
mock_cfg = MagicMock(spec=ConfigService)
|
|
|
|
def _resolve(key: str, **kwargs: Any) -> ResolvedValue:
|
|
if key == "actor.default.estimation":
|
|
return ResolvedValue(
|
|
key=key,
|
|
value=actor_name,
|
|
source=ConfigLevel.GLOBAL,
|
|
chain=[],
|
|
)
|
|
# For any other key, return a ResolvedValue with None
|
|
return ResolvedValue(key=key, value=None, source=ConfigLevel.DEFAULT, chain=[])
|
|
|
|
mock_cfg.resolve.side_effect = _resolve
|
|
return mock_cfg # type: ignore[return-value]
|
|
|
|
|
|
def _make_mock_config_service_no_estimation() -> MagicMock:
|
|
"""Create a mock ConfigService that returns None for actor.default.estimation."""
|
|
from cleveragents.application.services.config_service import (
|
|
ConfigLevel,
|
|
ResolvedValue,
|
|
)
|
|
|
|
mock_cfg = MagicMock(spec=ConfigService)
|
|
|
|
def _resolve(key: str, **kwargs: Any) -> ResolvedValue:
|
|
return ResolvedValue(key=key, value=None, source=ConfigLevel.DEFAULT, chain=[])
|
|
|
|
mock_cfg.resolve.side_effect = _resolve
|
|
return mock_cfg # type: ignore[return-value]
|
|
|
|
|
|
def _drive_to_execute(svc: PlanLifecycleService, plan_id: str) -> None:
|
|
"""Drive a plan from STRATEGIZE/QUEUED through to Execute phase."""
|
|
svc.start_strategize(plan_id)
|
|
plan = svc.get_plan(plan_id)
|
|
plan.processing_state = ProcessingState.PROCESSING
|
|
svc._commit_plan(plan)
|
|
svc.complete_strategize(plan_id)
|
|
# complete_strategize calls auto_progress; if still in STRATEGIZE/COMPLETE
|
|
# we need to call execute_plan explicitly.
|
|
plan = svc.get_plan(plan_id)
|
|
if plan.phase == PlanPhase.STRATEGIZE:
|
|
svc.execute_plan(plan_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subtask 1: 4-level fallback chain
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I have a lifecycle service with a global estimation actor config "{actor}"')
|
|
def step_service_with_global_estimation(context: Context, actor: str) -> None:
|
|
"""Create a lifecycle service with actor.default.estimation set in config."""
|
|
cfg = _make_mock_config_service_with_estimation(actor)
|
|
context.fallback_svc = _make_service(config_service=cfg)
|
|
context.fallback_error = None
|
|
|
|
|
|
@given("I have a lifecycle service without a global estimation actor config")
|
|
def step_service_without_global_estimation(context: Context) -> None:
|
|
"""Create a lifecycle service with no actor.default.estimation config."""
|
|
cfg = _make_mock_config_service_no_estimation()
|
|
context.fallback_svc = _make_service(config_service=cfg)
|
|
context.fallback_error = None
|
|
|
|
|
|
@given('I create an action with estimation_actor "{actor}"')
|
|
def step_create_action_with_estimation_actor(context: Context, actor: str) -> None:
|
|
"""Create an action with the given estimation_actor."""
|
|
context.fallback_svc.create_action(
|
|
name="local/fallback-test",
|
|
description="Fallback chain test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor=actor,
|
|
)
|
|
|
|
|
|
@given("I create an action without an estimation_actor")
|
|
def step_create_action_without_estimation_actor(context: Context) -> None:
|
|
"""Create an action with no estimation_actor."""
|
|
context.fallback_svc.create_action(
|
|
name="local/fallback-test",
|
|
description="Fallback chain test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
|
|
@when("I use the action to create a plan")
|
|
def step_use_action_create_plan(context: Context) -> None:
|
|
"""Use the action to create a plan."""
|
|
context.fallback_plan = context.fallback_svc.use_action(
|
|
action_name="local/fallback-test"
|
|
)
|
|
|
|
|
|
@then('the plan estimation_actor should be "{expected}"')
|
|
def step_check_plan_estimation_actor(context: Context, expected: str) -> None:
|
|
"""Verify the plan's estimation_actor matches the expected value."""
|
|
assert context.fallback_plan.estimation_actor == expected, (
|
|
f"Expected estimation_actor={expected!r}, "
|
|
f"got {context.fallback_plan.estimation_actor!r}"
|
|
)
|
|
|
|
|
|
@then("the plan estimation_actor should be None for fallback tests")
|
|
def step_check_plan_estimation_actor_none(context: Context) -> None:
|
|
"""Verify the plan's estimation_actor is None."""
|
|
assert context.fallback_plan.estimation_actor is None, (
|
|
f"Expected estimation_actor=None, "
|
|
f"got {context.fallback_plan.estimation_actor!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subtask 2: PLAN_ESTIMATION_COMPLETE event
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a lifecycle service with an event bus for estimation tests")
|
|
def step_service_with_event_bus(context: Context) -> None:
|
|
"""Create a lifecycle service with a mock event bus."""
|
|
context.emitted_events = [] # type: ignore[attr-defined]
|
|
|
|
mock_bus = MagicMock()
|
|
|
|
def _capture_emit(event: Any) -> None:
|
|
context.emitted_events.append(event)
|
|
|
|
mock_bus.emit.side_effect = _capture_emit
|
|
context.event_svc = _make_service(event_bus=mock_bus)
|
|
|
|
|
|
@given('I create an action with estimation_actor "{actor}" for event tests')
|
|
def step_create_action_for_event_tests(context: Context, actor: str) -> None:
|
|
"""Create an action with estimation_actor for event emission tests."""
|
|
context.event_svc.create_action(
|
|
name="local/event-test",
|
|
description="Event test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor=actor,
|
|
)
|
|
|
|
|
|
@given("I create an action without estimation_actor for event tests")
|
|
def step_create_action_no_estimation_for_event_tests(context: Context) -> None:
|
|
"""Create an action without estimation_actor for event emission tests."""
|
|
context.event_svc.create_action(
|
|
name="local/event-test-no-est",
|
|
description="Event test action without estimation",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
|
|
@when("I use the action and drive through strategize to execute")
|
|
def step_drive_through_strategize_to_execute(context: Context) -> None:
|
|
"""Drive the plan from use_action through strategize to execute."""
|
|
plan = context.event_svc.use_action(action_name="local/event-test")
|
|
context.event_plan = plan
|
|
_drive_to_execute(context.event_svc, plan.identity.plan_id)
|
|
context.event_plan = context.event_svc.get_plan(plan.identity.plan_id)
|
|
|
|
|
|
@when("I use the action and drive through strategize to execute without estimation")
|
|
def step_drive_through_strategize_to_execute_no_estimation(context: Context) -> None:
|
|
"""Drive the plan from use_action through strategize to execute (no estimation)."""
|
|
plan = context.event_svc.use_action(action_name="local/event-test-no-est")
|
|
context.event_plan = plan
|
|
_drive_to_execute(context.event_svc, plan.identity.plan_id)
|
|
context.event_plan = context.event_svc.get_plan(plan.identity.plan_id)
|
|
|
|
|
|
@then("a PLAN_ESTIMATION_COMPLETE event should have been emitted")
|
|
def step_check_estimation_complete_event_emitted(context: Context) -> None:
|
|
"""Verify that a PLAN_ESTIMATION_COMPLETE event was emitted."""
|
|
event_types = [getattr(e, "event_type", None) for e in context.emitted_events]
|
|
assert EventType.PLAN_ESTIMATION_COMPLETE in event_types, (
|
|
f"Expected PLAN_ESTIMATION_COMPLETE in emitted events, got: {event_types}"
|
|
)
|
|
|
|
|
|
@then("no PLAN_ESTIMATION_COMPLETE event should have been emitted")
|
|
def step_check_no_estimation_complete_event(context: Context) -> None:
|
|
"""Verify that no PLAN_ESTIMATION_COMPLETE event was emitted."""
|
|
event_types = [getattr(e, "event_type", None) for e in context.emitted_events]
|
|
assert EventType.PLAN_ESTIMATION_COMPLETE not in event_types, (
|
|
f"Expected no PLAN_ESTIMATION_COMPLETE event, but found one in: {event_types}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subtask 3: Conditional estimation step in Strategize-to-Execute
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a plan lifecycle service for estimation lifecycle tests")
|
|
def step_service_for_lifecycle_tests(context: Context) -> None:
|
|
"""Create a plan lifecycle service for lifecycle tests."""
|
|
context.lifecycle_svc = _make_service()
|
|
context.lifecycle_error = None
|
|
|
|
|
|
@given('I create an action with estimation_actor "{actor}" for lifecycle tests')
|
|
def step_create_action_with_estimation_for_lifecycle(
|
|
context: Context, actor: str
|
|
) -> None:
|
|
"""Create an action with estimation_actor for lifecycle tests."""
|
|
context.lifecycle_svc.create_action(
|
|
name="local/lifecycle-est-test",
|
|
description="Lifecycle estimation test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor=actor,
|
|
)
|
|
|
|
|
|
@given("I create an action without estimation_actor for lifecycle tests")
|
|
def step_create_action_without_estimation_for_lifecycle(context: Context) -> None:
|
|
"""Create an action without estimation_actor for lifecycle tests."""
|
|
context.lifecycle_svc.create_action(
|
|
name="local/lifecycle-no-est-test",
|
|
description="Lifecycle no-estimation test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
|
|
@when("I use the action and complete strategize for lifecycle tests")
|
|
def step_use_and_complete_strategize_lifecycle(context: Context) -> None:
|
|
"""Use the action and complete the strategize phase."""
|
|
# Try both action names
|
|
try:
|
|
plan = context.lifecycle_svc.use_action(action_name="local/lifecycle-est-test")
|
|
except Exception:
|
|
plan = context.lifecycle_svc.use_action(
|
|
action_name="local/lifecycle-no-est-test"
|
|
)
|
|
context.lifecycle_plan = plan
|
|
plan_id = plan.identity.plan_id
|
|
context.lifecycle_svc.start_strategize(plan_id)
|
|
p = context.lifecycle_svc.get_plan(plan_id)
|
|
p.processing_state = ProcessingState.PROCESSING
|
|
context.lifecycle_svc._commit_plan(p)
|
|
context.lifecycle_svc.complete_strategize(plan_id)
|
|
context.lifecycle_plan = context.lifecycle_svc.get_plan(plan_id)
|
|
|
|
|
|
@when("I call execute_plan for lifecycle tests")
|
|
def step_call_execute_plan_lifecycle(context: Context) -> None:
|
|
"""Call execute_plan if the plan is still in STRATEGIZE phase."""
|
|
plan = context.lifecycle_plan
|
|
if plan.phase == PlanPhase.STRATEGIZE:
|
|
context.lifecycle_plan = context.lifecycle_svc.execute_plan(
|
|
plan.identity.plan_id
|
|
)
|
|
else:
|
|
context.lifecycle_plan = context.lifecycle_svc.get_plan(plan.identity.plan_id)
|
|
|
|
|
|
@then("the plan should be in Execute phase")
|
|
def step_check_plan_in_execute_phase(context: Context) -> None:
|
|
"""Verify the plan is in Execute phase."""
|
|
assert context.lifecycle_plan.phase == PlanPhase.EXECUTE, (
|
|
f"Expected Execute phase, got {context.lifecycle_plan.phase}"
|
|
)
|
|
|
|
|
|
@then("the plan estimation_result should be populated")
|
|
def step_check_estimation_result_populated(context: Context) -> None:
|
|
"""Verify the plan has an estimation result."""
|
|
assert context.lifecycle_plan.estimation_result is not None, (
|
|
"Expected estimation_result to be set, but it is None"
|
|
)
|
|
|
|
|
|
@then("the plan estimation_result should be None for lifecycle tests")
|
|
def step_check_estimation_result_none_lifecycle(context: Context) -> None:
|
|
"""Verify the plan has no estimation result."""
|
|
assert context.lifecycle_plan.estimation_result is None, (
|
|
f"Expected estimation_result=None, "
|
|
f"got {context.lifecycle_plan.estimation_result!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subtask 4: cost_estimate_usd populated from estimation result
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a plan lifecycle service for cost estimate tests")
|
|
def step_service_for_cost_tests(context: Context) -> None:
|
|
"""Create a plan lifecycle service for cost estimate tests."""
|
|
context.cost_svc = _make_service()
|
|
|
|
|
|
@given('I create an action with estimation_actor "{actor}" for cost tests')
|
|
def step_create_action_for_cost_tests(context: Context, actor: str) -> None:
|
|
"""Create an action with estimation_actor for cost tests."""
|
|
context.cost_svc.create_action(
|
|
name="local/cost-test",
|
|
description="Cost estimate test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor=actor,
|
|
)
|
|
|
|
|
|
@when("I use the action and drive through strategize to execute for cost tests")
|
|
def step_drive_to_execute_for_cost_tests(context: Context) -> None:
|
|
"""Drive the plan through strategize to execute for cost tests."""
|
|
plan = context.cost_svc.use_action(action_name="local/cost-test")
|
|
context.cost_plan = plan
|
|
_drive_to_execute(context.cost_svc, plan.identity.plan_id)
|
|
context.cost_plan = context.cost_svc.get_plan(plan.identity.plan_id)
|
|
|
|
|
|
@then("the plan cost_estimate_usd should be set")
|
|
def step_check_cost_estimate_usd_set(context: Context) -> None:
|
|
"""Verify cost_estimate_usd is populated on the plan."""
|
|
# The stub actor may or may not set estimated_cost_usd; check that
|
|
# if estimation_result is set, cost_estimate_usd is consistent.
|
|
plan = context.cost_plan
|
|
if (
|
|
plan.estimation_result is not None
|
|
and plan.estimation_result.estimated_cost_usd is not None
|
|
):
|
|
assert plan.cost_estimate_usd == plan.estimation_result.estimated_cost_usd, (
|
|
f"cost_estimate_usd={plan.cost_estimate_usd!r} does not match "
|
|
f"estimation_result.estimated_cost_usd="
|
|
f"{plan.estimation_result.estimated_cost_usd!r}"
|
|
)
|
|
# If estimated_cost_usd is None in the result, cost_estimate_usd may be None
|
|
# Either way, no assertion error means the wiring is correct
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subtask 5: Clean skip when no estimation actor configured
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a plan lifecycle service for skip tests")
|
|
def step_service_for_skip_tests(context: Context) -> None:
|
|
"""Create a plan lifecycle service for skip tests."""
|
|
context.skip_svc = _make_service()
|
|
|
|
|
|
@given("I create an action without estimation_actor for skip tests")
|
|
def step_create_action_without_estimation_for_skip(context: Context) -> None:
|
|
"""Create an action without estimation_actor for skip tests."""
|
|
context.skip_svc.create_action(
|
|
name="local/skip-test",
|
|
description="Skip estimation test action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
|
|
@when("I use the action and complete strategize for skip tests")
|
|
def step_use_and_complete_strategize_skip(context: Context) -> None:
|
|
"""Use the action and complete strategize for skip tests."""
|
|
plan = context.skip_svc.use_action(action_name="local/skip-test")
|
|
context.skip_plan = plan
|
|
plan_id = plan.identity.plan_id
|
|
context.skip_svc.start_strategize(plan_id)
|
|
p = context.skip_svc.get_plan(plan_id)
|
|
p.processing_state = ProcessingState.PROCESSING
|
|
context.skip_svc._commit_plan(p)
|
|
context.skip_svc.complete_strategize(plan_id)
|
|
context.skip_plan = context.skip_svc.get_plan(plan_id)
|
|
|
|
|
|
@when("I call execute_plan for skip tests")
|
|
def step_call_execute_plan_skip(context: Context) -> None:
|
|
"""Call execute_plan for skip tests."""
|
|
plan = context.skip_plan
|
|
if plan.phase == PlanPhase.STRATEGIZE:
|
|
context.skip_plan = context.skip_svc.execute_plan(plan.identity.plan_id)
|
|
else:
|
|
context.skip_plan = context.skip_svc.get_plan(plan.identity.plan_id)
|
|
|
|
|
|
@then("the plan should transition to Execute phase cleanly")
|
|
def step_check_plan_execute_phase_cleanly(context: Context) -> None:
|
|
"""Verify the plan is in Execute phase."""
|
|
assert context.skip_plan.phase == PlanPhase.EXECUTE, (
|
|
f"Expected Execute phase, got {context.skip_plan.phase}"
|
|
)
|
|
|
|
|
|
@then("no estimation result should be set on the plan")
|
|
def step_check_no_estimation_result_skip(context: Context) -> None:
|
|
"""Verify no estimation result is set."""
|
|
assert context.skip_plan.estimation_result is None, (
|
|
f"Expected estimation_result=None, got {context.skip_plan.estimation_result!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# EventType enum check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the EventType enum should have PLAN_ESTIMATION_COMPLETE")
|
|
def step_check_event_type_has_estimation_complete(context: Context) -> None:
|
|
"""Verify EventType has PLAN_ESTIMATION_COMPLETE."""
|
|
assert hasattr(EventType, "PLAN_ESTIMATION_COMPLETE"), (
|
|
"EventType does not have PLAN_ESTIMATION_COMPLETE"
|
|
)
|
|
|
|
|
|
@then('the PLAN_ESTIMATION_COMPLETE value should be "plan.estimation_complete"')
|
|
def step_check_estimation_complete_value(context: Context) -> None:
|
|
"""Verify PLAN_ESTIMATION_COMPLETE value."""
|
|
assert EventType.PLAN_ESTIMATION_COMPLETE == "plan.estimation_complete", (
|
|
f"Expected 'plan.estimation_complete', "
|
|
f"got {EventType.PLAN_ESTIMATION_COMPLETE!r}"
|
|
)
|