forked from HAL9000/cleveragents-core
78e146d169
Implements all four automatic checkpoint triggers defined in the specification for the Execute phase of the plan lifecycle: - before_tool_execute: Checkpoint created before any tool with writes=True runs - after_tool_execute: Checkpoint created after a write tool completes successfully - on_subplan_spawn: Checkpoint created immediately after a child plan is spawned - on_error: Checkpoint created after any unrecoverable error in the Execute phase Changes: - tool/runner.py: Added optional CheckpointService and auto_checkpoint_triggers parameters to ToolRunner. Checkpoint hooks fire around write-tool execution when a CheckpointService is wired. Exported DEFAULT_AUTO_TRIGGERS as a public constant (single source of truth). Made is_trigger_active() public so callers can query the active trigger set without accessing private attributes. - application/services/subplan_execution_service.py: Added optional CheckpointService, auto_checkpoint_triggers, and parent_plan_id parameters. on_subplan_spawn checkpoint fires in _execute_one_with_retry before the first execution attempt. Now imports DEFAULT_AUTO_TRIGGERS from runner.py (DRY fix). - application/services/plan_executor.py: Added _is_auto_trigger_active() helper and on_error checkpoint hooks in both _run_execute_with_stub() and _run_execute_with_runtime() error paths. Delegates to ToolRunner.is_trigger_active() instead of accessing private attributes (module boundary fix). - application/services/config_service.py: Registered new config key core.checkpoints.auto_create_on (default: all four triggers enabled) with env var CLEVERAGENTS_CHECKPOINT_AUTO_CREATE_ON. - application/services/llm_actors.py: Replaced Any type for lifecycle_service with PlanLifecycleProtocol (typed Protocol) and tool_runner with ToolRunner type annotation. Eliminates Any usage for injected dependencies. Tests: - features/checkpoint_auto_triggers.feature: 15 Behave scenarios covering all four triggers, disable-trigger behavior, no-checkpoint-service fallback, and config key registration. - features/steps/checkpoint_auto_triggers_tool_steps.py: Step definitions for ToolRunner and config service scenarios (split from original 519-line file). - features/steps/checkpoint_auto_triggers_executor_steps.py: Step definitions for SubplanExecutionService and PlanExecutor scenarios (split from original). Closes #3439 ISSUES CLOSED: #3439
332 lines
11 KiB
Python
332 lines
11 KiB
Python
"""Step definitions for checkpoint_auto_triggers.feature — executor steps.
|
|
|
|
Covers automatic checkpoint creation for:
|
|
- on_subplan_spawn (SubplanExecutionService)
|
|
- on_error (PlanExecutor)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.application.services.checkpoint_service import CheckpointService
|
|
from cleveragents.application.services.subplan_execution_service import (
|
|
SubplanExecutionOutput,
|
|
SubplanExecutionService,
|
|
)
|
|
from cleveragents.domain.models.core.plan import (
|
|
ExecutionMode,
|
|
PlanPhase,
|
|
ProcessingState,
|
|
SubplanConfig,
|
|
SubplanMergeStrategy,
|
|
SubplanStatus,
|
|
)
|
|
from cleveragents.tool.registry import ToolRegistry
|
|
from cleveragents.tool.runner import DEFAULT_AUTO_TRIGGERS, ToolRunner
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_plan_mock(
|
|
plan_id: str,
|
|
phase: PlanPhase = PlanPhase.EXECUTE,
|
|
state: ProcessingState = ProcessingState.QUEUED,
|
|
decision_root_id: str = "01ROOT000000000000000000001",
|
|
) -> MagicMock:
|
|
"""Build a mock plan object."""
|
|
from cleveragents.domain.models.core.plan import PlanTimestamps
|
|
|
|
plan = MagicMock()
|
|
plan.phase = phase
|
|
plan.state = state
|
|
plan.decision_root_id = decision_root_id
|
|
plan.definition_of_done = "Step one"
|
|
plan.invariants = []
|
|
plan.timestamps = PlanTimestamps()
|
|
plan.changeset_id = None
|
|
plan.sandbox_refs = []
|
|
plan.error_details = None
|
|
plan.read_only = False
|
|
return plan
|
|
|
|
|
|
def _make_lifecycle_mock(plan: Any) -> MagicMock:
|
|
"""Build a mock lifecycle service."""
|
|
lcs = MagicMock()
|
|
lcs.get_plan.return_value = plan
|
|
lcs.start_execute = MagicMock()
|
|
lcs.complete_execute = MagicMock()
|
|
lcs.fail_execute = MagicMock()
|
|
lcs._commit_plan = MagicMock()
|
|
return lcs
|
|
|
|
|
|
def _make_subplan_status(subplan_id: str) -> SubplanStatus:
|
|
"""Build a SubplanStatus for testing."""
|
|
return SubplanStatus(
|
|
subplan_id=subplan_id,
|
|
action_name="test/action",
|
|
status=ProcessingState.QUEUED,
|
|
)
|
|
|
|
|
|
def _make_subplan_config() -> SubplanConfig:
|
|
"""Build a minimal SubplanConfig."""
|
|
return SubplanConfig(
|
|
execution_mode=ExecutionMode.SEQUENTIAL,
|
|
merge_strategy=SubplanMergeStrategy.LAST_WINS,
|
|
max_parallel=1,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps — SubplanExecutionService
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given(
|
|
'a subplan execution service with a checkpoint service and parent_plan_id "{plan_id}"'
|
|
)
|
|
def step_subplan_service_with_checkpoint(context: Context, plan_id: str) -> None:
|
|
context.checkpoint_service = CheckpointService()
|
|
context.parent_plan_id = plan_id
|
|
context.subplan_ids: list[str] = []
|
|
context.disabled_subplan_triggers: set[str] = set()
|
|
|
|
|
|
@given("a subplan execution service WITHOUT a checkpoint service")
|
|
def step_subplan_service_without_checkpoint(context: Context) -> None:
|
|
context.checkpoint_service = None
|
|
context.parent_plan_id = ""
|
|
context.subplan_ids = []
|
|
context.disabled_subplan_triggers = set()
|
|
|
|
|
|
@given('a subplan "{subplan_id}" is configured')
|
|
def step_configure_subplan(context: Context, subplan_id: str) -> None:
|
|
context.subplan_ids = [subplan_id]
|
|
|
|
|
|
@given("{count:d} subplans are configured")
|
|
def step_configure_multiple_subplans(context: Context, count: int) -> None:
|
|
from ulid import ULID
|
|
|
|
context.subplan_ids = [str(ULID()) for _ in range(count)]
|
|
|
|
|
|
@given('the "{trigger}" trigger is disabled on the subplan service')
|
|
def step_disable_subplan_trigger(context: Context, trigger: str) -> None:
|
|
if not hasattr(context, "disabled_subplan_triggers"):
|
|
context.disabled_subplan_triggers = set()
|
|
context.disabled_subplan_triggers.add(trigger)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps — SubplanExecutionService
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _build_subplan_service(context: Context) -> SubplanExecutionService:
|
|
"""Build the SubplanExecutionService with optional checkpoint service."""
|
|
active_triggers = DEFAULT_AUTO_TRIGGERS - getattr(
|
|
context, "disabled_subplan_triggers", set()
|
|
)
|
|
|
|
def _executor_fn(status: SubplanStatus) -> SubplanExecutionOutput:
|
|
return SubplanExecutionOutput(
|
|
subplan_id=status.subplan_id,
|
|
success=True,
|
|
files={},
|
|
)
|
|
|
|
return SubplanExecutionService(
|
|
config=_make_subplan_config(),
|
|
executor_fn=_executor_fn,
|
|
checkpoint_service=context.checkpoint_service,
|
|
auto_checkpoint_triggers=frozenset(active_triggers),
|
|
parent_plan_id=getattr(context, "parent_plan_id", ""),
|
|
)
|
|
|
|
|
|
@when("the subplan execution service executes the subplan")
|
|
def step_execute_subplan(context: Context) -> None:
|
|
service = _build_subplan_service(context)
|
|
subplan_ids = getattr(context, "subplan_ids", [])
|
|
statuses = [_make_subplan_status(sid) for sid in subplan_ids]
|
|
try:
|
|
context.subplan_result = service.execute_all(statuses, base_files={})
|
|
context.exception = None
|
|
except Exception as exc:
|
|
context.exception = exc
|
|
|
|
|
|
@when("the subplan execution service executes all subplans")
|
|
def step_execute_all_subplans(context: Context) -> None:
|
|
step_execute_subplan(context)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps — SubplanExecutionService checkpoints
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('the checkpoint source_tool should be "subplan_execution_service"')
|
|
def step_checkpoint_source_tool(context: Context) -> None:
|
|
plan_id = context.parent_plan_id
|
|
checkpoints = context.checkpoint_service.list_checkpoints(plan_id)
|
|
assert checkpoints, f"No checkpoints found for plan {plan_id}"
|
|
source_tools = [cp.metadata.source_tool for cp in checkpoints]
|
|
assert "subplan_execution_service" in source_tools, (
|
|
f"Expected source_tool 'subplan_execution_service', got: {source_tools}"
|
|
)
|
|
|
|
|
|
@then('no checkpoints should exist for plan "{plan_id}"')
|
|
def step_no_checkpoints_for_plan(context: Context, plan_id: str) -> None:
|
|
checkpoints = context.checkpoint_service.list_checkpoints(plan_id)
|
|
assert len(checkpoints) == 0, (
|
|
f"Expected 0 checkpoints for plan {plan_id}, got {len(checkpoints)}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps — PlanExecutor on_error
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a plan executor with a checkpoint manager and plan_id "{plan_id}"')
|
|
def step_plan_executor_with_checkpoint(context: Context, plan_id: str) -> None:
|
|
from cleveragents.infrastructure.sandbox.checkpoint import CheckpointManager
|
|
|
|
context.plan_id = plan_id
|
|
context.checkpoint_manager = MagicMock(spec=CheckpointManager)
|
|
context.checkpoint_manager.create_checkpoint = MagicMock(return_value=MagicMock())
|
|
context.checkpoint_manager.list_checkpoints = MagicMock(return_value=[])
|
|
context.disabled_executor_triggers: set[str] = set()
|
|
context.error_actor_raises = False
|
|
|
|
|
|
@given("the execute actor raises an error")
|
|
def step_execute_actor_raises(context: Context) -> None:
|
|
context.error_actor_raises = True
|
|
|
|
|
|
@given('the "{trigger}" trigger is disabled on the executor')
|
|
def step_disable_executor_trigger(context: Context, trigger: str) -> None:
|
|
if not hasattr(context, "disabled_executor_triggers"):
|
|
context.disabled_executor_triggers = set()
|
|
context.disabled_executor_triggers.add(trigger)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps — PlanExecutor on_error
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run the execute phase for plan "{plan_id}"')
|
|
def step_run_execute_phase(context: Context, plan_id: str) -> None:
|
|
from cleveragents.application.services.plan_executor import PlanExecutor
|
|
|
|
active_triggers = DEFAULT_AUTO_TRIGGERS - getattr(
|
|
context, "disabled_executor_triggers", set()
|
|
)
|
|
|
|
plan = _make_plan_mock(plan_id)
|
|
lifecycle = _make_lifecycle_mock(plan)
|
|
|
|
# Build a tool runner with the active triggers (used by _is_auto_trigger_active)
|
|
registry = ToolRegistry()
|
|
tool_runner = ToolRunner(
|
|
registry=registry,
|
|
auto_checkpoint_triggers=frozenset(active_triggers),
|
|
)
|
|
|
|
# Build a failing execute actor if requested
|
|
if getattr(context, "error_actor_raises", False):
|
|
|
|
class _FailingActor:
|
|
def execute(self, **kwargs: Any) -> None:
|
|
raise RuntimeError("Simulated execute failure")
|
|
|
|
execute_actor: Any = _FailingActor()
|
|
else:
|
|
execute_actor = None
|
|
|
|
executor = PlanExecutor(
|
|
lifecycle_service=lifecycle,
|
|
tool_runner=tool_runner,
|
|
checkpoint_manager=context.checkpoint_manager,
|
|
execute_actor=execute_actor,
|
|
sandbox_root="/tmp/test-sandbox-auto-trigger",
|
|
)
|
|
|
|
try:
|
|
executor.run_execute(plan_id)
|
|
context.execute_exception = None
|
|
except Exception as exc:
|
|
context.execute_exception = exc
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps — PlanExecutor on_error
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the execute phase should fail")
|
|
def step_execute_phase_failed(context: Context) -> None:
|
|
assert context.execute_exception is not None, (
|
|
"Expected execute phase to fail, but it succeeded"
|
|
)
|
|
|
|
|
|
@then(
|
|
'a checkpoint with phase "on_error" should have been created for plan "{plan_id}"'
|
|
)
|
|
def step_on_error_checkpoint_created(context: Context, plan_id: str) -> None:
|
|
# The checkpoint manager mock records calls to create_checkpoint
|
|
calls = context.checkpoint_manager.create_checkpoint.call_args_list
|
|
|
|
# Check that create_checkpoint was called with phase="on_error"
|
|
assert context.checkpoint_manager.create_checkpoint.called, (
|
|
"Expected create_checkpoint to be called, but it was not"
|
|
)
|
|
# Verify at least one call had phase="on_error"
|
|
found = False
|
|
for call in calls:
|
|
args = call.args
|
|
kwargs = call.kwargs
|
|
if kwargs.get("phase") == "on_error":
|
|
found = True
|
|
break
|
|
if len(args) > 2 and args[2] == "on_error":
|
|
found = True
|
|
break
|
|
assert found, (
|
|
f"Expected a create_checkpoint call with phase='on_error'. Calls: {calls}"
|
|
)
|
|
|
|
|
|
@then('no on_error checkpoint should have been created for plan "{plan_id}"')
|
|
def step_no_on_error_checkpoint(context: Context, plan_id: str) -> None:
|
|
calls = context.checkpoint_manager.create_checkpoint.call_args_list
|
|
found = False
|
|
for call in calls:
|
|
args = call.args
|
|
kwargs = call.kwargs
|
|
if kwargs.get("phase") == "on_error":
|
|
found = True
|
|
break
|
|
if len(args) > 2 and args[2] == "on_error":
|
|
found = True
|
|
break
|
|
assert not found, (
|
|
f"Expected NO create_checkpoint call with phase='on_error'. Calls: {calls}"
|
|
)
|