forked from HAL9000/cleveragents-core
007af498b8
Renamed all 11 task-type confidence threshold fields in AutomationProfile from phase-transition semantics to spec-defined task-type semantics. Updated all 8 built-in profiles, CLI formatting, YAML schema, services, and all Behave/Robot tests referencing the old field names. Post-review fixes: - Fixed 24 stale old field names in M6 fixture files (automation_profiles.json, autonomy_guardrails.json) - Added model_validator(mode='before') to detect legacy field names and raise actionable ValueError with rename mapping - Added semantic bridge comments in PlanLifecycleService mapping task-type thresholds to phase-transition gates - Added threshold_field to structured log messages for observability - Restored categorised CLI automation-profile show output to match spec (Phase Transitions / Decision Automation / Self-Repair / Execution Controls) instead of flat list - Added missing access_network field to spec show output examples (Rich, Plain, JSON, YAML variants) - Aligned ADR-017 profile fields table to all 11 fields with descriptions matching spec Automatable Tasks table - Aligned automation_profiles.md threshold descriptions with spec - Added spec section references in phase_reversion.md, error_recovery.md, and plan_execute.md for field naming context - Extended repository roundtrip test to assert all 11 threshold fields - Fixed benchmark _make_profile() passing safety fields as top-level kwargs instead of via SafetyProfile sub-model (incompatible with extra="forbid") - Aligned CLI JSON/YAML output structure for automation-profile show with the specification grouped format (phase_transitions, decision_automation, self_repair, execution_controls) - Moved safety boolean fields into the Execution Controls section of Rich output per spec examples - Reverted auto profile description to "Fully automatic except apply" per specification (line 16703, line 28406) - Improved bridge comments in test steps with semantic context for threshold-to-gate mappings ISSUES CLOSED: #902
500 lines
20 KiB
Python
500 lines
20 KiB
Python
"""Step definitions for plan_lifecycle_service_coverage_r2.feature.
|
|
|
|
Targets uncovered lines in PlanLifecycleService (build/coverage.xml hits=0):
|
|
- Lines 226-231: _try_record_decision exception handler
|
|
- Line 643: use_action description fallback to str(namespaced_name)
|
|
- Line 1437: try_auto_revert_from_apply early return (not Apply phase)
|
|
- Line 1440: try_auto_revert_from_apply early return (not CONSTRAINED)
|
|
- Lines 1452-1458: try_auto_revert_from_apply loop guard (MAX_REVERSIONS)
|
|
- Line 1488: try_auto_revert_from_execute early return (not Execute phase)
|
|
- Lines 1500-1506: try_auto_revert_from_execute loop guard (MAX_REVERSIONS)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tempfile
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.application.services.plan_lifecycle_service import (
|
|
PlanLifecycleService,
|
|
)
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.domain.models.core import Actor
|
|
from cleveragents.domain.models.core.plan import (
|
|
AutomationProfileProvenance,
|
|
AutomationProfileRef,
|
|
PlanPhase,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
)
|
|
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
|
|
|
|
# -----------------------------------------------------------------
|
|
# Background
|
|
# -----------------------------------------------------------------
|
|
|
|
|
|
@given("I have a fresh plan lifecycle service for coverage r2")
|
|
def step_create_fresh_service_r2(context: Context) -> None:
|
|
"""Create a clean PlanLifecycleService for round-2 coverage tests."""
|
|
Settings._instance = None
|
|
settings = Settings()
|
|
context.service = PlanLifecycleService(settings=settings)
|
|
context.error = None
|
|
|
|
|
|
# -----------------------------------------------------------------
|
|
# Helpers
|
|
# -----------------------------------------------------------------
|
|
|
|
|
|
def _create_action_r2(context: Context, name: str, **kwargs):
|
|
"""Create a basic action with sensible defaults."""
|
|
defaults = {
|
|
"name": name,
|
|
"description": f"Action {name}",
|
|
"definition_of_done": "Tests pass",
|
|
"strategy_actor": "openai/gpt-4",
|
|
"execution_actor": "openai/gpt-4",
|
|
}
|
|
defaults.update(kwargs)
|
|
return context.service.create_action(**defaults)
|
|
|
|
|
|
def _create_plan_in_phase_r2(
|
|
context: Context,
|
|
target_phase: PlanPhase,
|
|
action_name: str | None = None,
|
|
) -> object:
|
|
"""Create a plan and advance it to the given phase."""
|
|
aname = action_name or f"local/cov-r2-{id(context)}"
|
|
action = _create_action_r2(context, aname)
|
|
plan = context.service.use_action(
|
|
action_name=str(action.namespaced_name),
|
|
project_links=[ProjectLink(project_name="proj-r2")],
|
|
)
|
|
pid = plan.identity.plan_id
|
|
|
|
if target_phase in (PlanPhase.EXECUTE, PlanPhase.APPLY):
|
|
context.service.start_strategize(pid)
|
|
context.service.complete_strategize(pid)
|
|
# complete_strategize calls auto_progress which may call execute_plan
|
|
# Refresh the plan
|
|
plan = context.service.get_plan(pid)
|
|
if plan.phase == PlanPhase.STRATEGIZE:
|
|
context.service.execute_plan(pid)
|
|
|
|
if target_phase == PlanPhase.APPLY:
|
|
plan = context.service.get_plan(pid)
|
|
if plan.phase == PlanPhase.EXECUTE:
|
|
context.service.start_execute(pid)
|
|
context.service.complete_execute(pid)
|
|
# complete_execute calls auto_progress which may call apply_plan
|
|
plan = context.service.get_plan(pid)
|
|
if plan.phase == PlanPhase.EXECUTE:
|
|
context.service.apply_plan(pid)
|
|
|
|
context.plan = context.service.get_plan(pid)
|
|
return context.plan
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: Decision recording failure is silently caught (lines 226-231)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan lifecycle service with a failing decision service")
|
|
def step_create_service_with_failing_decision_service(context: Context) -> None:
|
|
"""Create a PlanLifecycleService with a DecisionService that raises."""
|
|
Settings._instance = None
|
|
settings = Settings()
|
|
|
|
# Create a mock decision service that always raises
|
|
mock_ds = MagicMock()
|
|
mock_ds.record_decision.side_effect = RuntimeError(
|
|
"Simulated decision recording failure"
|
|
)
|
|
|
|
context.service = PlanLifecycleService(
|
|
settings=settings,
|
|
decision_service=mock_ds,
|
|
)
|
|
context.mock_decision_service = mock_ds
|
|
context.error = None
|
|
|
|
|
|
@given('an action "{name}" exists for coverage r2')
|
|
def step_create_named_action_r2(context: Context, name: str) -> None:
|
|
"""Create an action with the given name."""
|
|
context.action = _create_action_r2(context, name)
|
|
|
|
|
|
@given('a plan created from "{action_name}" for coverage r2')
|
|
def step_create_plan_from_action_r2(context: Context, action_name: str) -> None:
|
|
"""Create a plan from the named action."""
|
|
context.plan = context.service.use_action(
|
|
action_name=action_name,
|
|
project_links=[ProjectLink(project_name="proj-r2")],
|
|
)
|
|
|
|
|
|
@when("I start strategize and the decision service raises an error")
|
|
def step_start_strategize_with_failing_ds(context: Context) -> None:
|
|
"""Start strategize — the decision service will raise, but it should
|
|
be caught internally and not propagate."""
|
|
context.error = None
|
|
try:
|
|
pid = context.plan.identity.plan_id
|
|
context.plan = context.service.start_strategize(pid)
|
|
except Exception as e:
|
|
context.error = e
|
|
|
|
|
|
@then("the plan should be in processing state without raising an error")
|
|
def step_verify_processing_state_no_error(context: Context) -> None:
|
|
"""Verify the plan transitioned to PROCESSING and no error propagated."""
|
|
assert context.error is None, f"Expected no error but got: {context.error}"
|
|
assert context.plan.processing_state == ProcessingState.PROCESSING, (
|
|
f"Expected PROCESSING, got {context.plan.processing_state}"
|
|
)
|
|
# Verify the mock was actually called (proving the except block ran)
|
|
assert context.mock_decision_service.record_decision.called, (
|
|
"Expected record_decision to have been called"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: use_action description fallback (line 643)
|
|
# =================================================================
|
|
|
|
|
|
@given('an action "{name}" with its description cleared after creation')
|
|
def step_create_action_with_blank_description(context: Context, name: str) -> None:
|
|
"""Create an action, then clear both description fields to force
|
|
the fallback to str(namespaced_name) on line 643."""
|
|
action = _create_action_r2(context, name)
|
|
# Bypass Pydantic validation to clear the description fields
|
|
action.__dict__["description"] = ""
|
|
action.__dict__["long_description"] = None
|
|
context.action = action
|
|
context.expected_fallback = str(action.namespaced_name)
|
|
|
|
|
|
@when("I use the blank-description action to create a plan for coverage r2")
|
|
def step_use_blank_desc_action(context: Context) -> None:
|
|
"""Use the action with cleared description fields."""
|
|
context.plan = context.service.use_action(
|
|
action_name=str(context.action.namespaced_name),
|
|
project_links=[ProjectLink(project_name="proj-blank")],
|
|
)
|
|
|
|
|
|
@then("the plan description should equal the action namespaced name string")
|
|
def step_verify_description_fallback(context: Context) -> None:
|
|
"""Verify the plan description fell back to str(namespaced_name)."""
|
|
assert context.plan.description == context.expected_fallback, (
|
|
f"Expected description '{context.expected_fallback}', "
|
|
f"got '{context.plan.description}'"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: try_auto_revert_from_apply not in Apply phase (line 1437)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan in strategize phase for auto-revert test")
|
|
def step_plan_in_strategize_for_revert(context: Context) -> None:
|
|
"""Create a plan that is still in Strategize phase."""
|
|
action = _create_action_r2(context, "local/revert-test-1")
|
|
context.plan = context.service.use_action(
|
|
action_name=str(action.namespaced_name),
|
|
project_links=[ProjectLink(project_name="proj-revert")],
|
|
)
|
|
|
|
|
|
@when("I call try_auto_revert_from_apply on the non-apply plan")
|
|
def step_try_revert_from_apply_wrong_phase(context: Context) -> None:
|
|
"""Call try_auto_revert_from_apply on a plan not in Apply phase."""
|
|
context.result_plan = context.service.try_auto_revert_from_apply(
|
|
context.plan.identity.plan_id,
|
|
reason="test revert",
|
|
)
|
|
|
|
|
|
@then("the plan should remain in strategize phase unchanged")
|
|
def step_verify_strategize_unchanged(context: Context) -> None:
|
|
"""Verify the plan is still in Strategize phase."""
|
|
assert context.result_plan.phase == PlanPhase.STRATEGIZE, (
|
|
f"Expected STRATEGIZE, got {context.result_plan.phase}"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: try_auto_revert_from_apply not constrained (line 1440)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan in apply phase with queued state for auto-revert test")
|
|
def step_plan_in_apply_queued(context: Context) -> None:
|
|
"""Create a plan in Apply/QUEUED state (not CONSTRAINED)."""
|
|
_create_plan_in_phase_r2(context, PlanPhase.APPLY)
|
|
# Plan should be in APPLY/QUEUED state after apply_plan
|
|
|
|
|
|
@when("I call try_auto_revert_from_apply on the non-constrained plan")
|
|
def step_try_revert_from_apply_not_constrained(context: Context) -> None:
|
|
"""Call try_auto_revert_from_apply on a plan that is not constrained."""
|
|
context.result_plan = context.service.try_auto_revert_from_apply(
|
|
context.plan.identity.plan_id,
|
|
reason="test revert not constrained",
|
|
)
|
|
|
|
|
|
@then("the plan should remain in apply phase with queued state")
|
|
def step_verify_apply_queued_unchanged(context: Context) -> None:
|
|
"""Verify the plan is still in Apply/QUEUED state."""
|
|
assert context.result_plan.phase == PlanPhase.APPLY, (
|
|
f"Expected APPLY, got {context.result_plan.phase}"
|
|
)
|
|
assert context.result_plan.processing_state == ProcessingState.QUEUED, (
|
|
f"Expected QUEUED, got {context.result_plan.processing_state}"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: try_auto_revert_from_apply loop guard (lines 1452-1458)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan in apply constrained state that has reached max reversions")
|
|
def step_plan_in_apply_constrained_max_reversions(context: Context) -> None:
|
|
"""Create a plan in Apply/CONSTRAINED with reversion_count >= MAX_REVERSIONS."""
|
|
_create_plan_in_phase_r2(context, PlanPhase.APPLY)
|
|
pid = context.plan.identity.plan_id
|
|
plan = context.service.get_plan(pid)
|
|
|
|
# Start and process the apply to get to a state where we can constrain
|
|
context.service.start_apply(pid)
|
|
context.service.constrain_apply(pid, "constraint hit")
|
|
|
|
# Set reversion_count to MAX_REVERSIONS to trigger the loop guard
|
|
plan = context.service.get_plan(pid)
|
|
plan.reversion_count = plan.MAX_REVERSIONS # 3
|
|
context.plan = plan
|
|
|
|
|
|
@given("the plan uses a profile that permits auto-reversion from apply")
|
|
def step_set_profile_permitting_reversion(context: Context) -> None:
|
|
"""Set an automation profile that allows auto-reversion from apply
|
|
(access_network < 1.0 permits automatic reversion)."""
|
|
context.plan.automation_profile = AutomationProfileRef(
|
|
profile_name="ci", # ci: access_network=0.0 (auto-revert from Apply permitted)
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
)
|
|
|
|
|
|
@when("I call try_auto_revert_from_apply on the max-reverted plan")
|
|
def step_try_revert_from_apply_max_reversions(context: Context) -> None:
|
|
"""Call try_auto_revert_from_apply on a plan at MAX_REVERSIONS."""
|
|
context.result_plan = context.service.try_auto_revert_from_apply(
|
|
context.plan.identity.plan_id,
|
|
reason="should be blocked by loop guard",
|
|
)
|
|
|
|
|
|
@then("the plan should remain in apply constrained state due to loop guard")
|
|
def step_verify_apply_constrained_blocked(context: Context) -> None:
|
|
"""Verify the plan is still in Apply/CONSTRAINED (reversion was blocked)."""
|
|
assert context.result_plan.phase == PlanPhase.APPLY, (
|
|
f"Expected APPLY, got {context.result_plan.phase}"
|
|
)
|
|
assert context.result_plan.processing_state == ProcessingState.CONSTRAINED, (
|
|
f"Expected CONSTRAINED, got {context.result_plan.processing_state}"
|
|
)
|
|
assert context.result_plan.reversion_count == context.result_plan.MAX_REVERSIONS, (
|
|
f"Expected reversion_count={context.result_plan.MAX_REVERSIONS}, "
|
|
f"got {context.result_plan.reversion_count}"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: try_auto_revert_from_execute not Execute phase (line 1488)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan in strategize phase for execute-revert test")
|
|
def step_plan_in_strategize_for_execute_revert(context: Context) -> None:
|
|
"""Create a plan in Strategize phase for the execute-revert test."""
|
|
action = _create_action_r2(context, "local/exec-revert-1")
|
|
context.plan = context.service.use_action(
|
|
action_name=str(action.namespaced_name),
|
|
project_links=[ProjectLink(project_name="proj-exec-revert")],
|
|
)
|
|
|
|
|
|
@when("I call try_auto_revert_from_execute on the non-execute plan")
|
|
def step_try_revert_from_execute_wrong_phase(context: Context) -> None:
|
|
"""Call try_auto_revert_from_execute on a plan not in Execute phase."""
|
|
context.result_plan = context.service.try_auto_revert_from_execute(
|
|
context.plan.identity.plan_id,
|
|
reason="test revert from non-execute",
|
|
)
|
|
|
|
|
|
@then("the plan should remain in strategize phase for execute-revert test")
|
|
def step_verify_strategize_unchanged_execute_revert(context: Context) -> None:
|
|
"""Verify the plan is still in Strategize phase."""
|
|
assert context.result_plan.phase == PlanPhase.STRATEGIZE, (
|
|
f"Expected STRATEGIZE, got {context.result_plan.phase}"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: try_auto_revert_from_execute loop guard (lines 1500-1506)
|
|
# =================================================================
|
|
|
|
|
|
@given("a plan in execute phase that has reached max reversions")
|
|
def step_plan_in_execute_max_reversions(context: Context) -> None:
|
|
"""Create a plan in Execute phase with reversion_count >= MAX_REVERSIONS."""
|
|
_create_plan_in_phase_r2(context, PlanPhase.EXECUTE)
|
|
pid = context.plan.identity.plan_id
|
|
plan = context.service.get_plan(pid)
|
|
|
|
# Set reversion_count to MAX_REVERSIONS to trigger the loop guard
|
|
plan.reversion_count = plan.MAX_REVERSIONS # 3
|
|
context.plan = plan
|
|
|
|
|
|
@given("the plan uses a profile that permits auto-reversion from execute")
|
|
def step_set_profile_permitting_execute_reversion(context: Context) -> None:
|
|
"""Set an automation profile that allows auto-reversion from execute
|
|
(delete_content < 1.0 permits strategy revision)."""
|
|
context.plan.automation_profile = AutomationProfileRef(
|
|
profile_name="ci", # ci: delete_content=0.0 (strategy revision permitted)
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
)
|
|
|
|
|
|
@when("I call try_auto_revert_from_execute on the max-reverted execute plan")
|
|
def step_try_revert_from_execute_max_reversions(context: Context) -> None:
|
|
"""Call try_auto_revert_from_execute on a plan at MAX_REVERSIONS."""
|
|
context.result_plan = context.service.try_auto_revert_from_execute(
|
|
context.plan.identity.plan_id,
|
|
reason="should be blocked by loop guard",
|
|
)
|
|
|
|
|
|
@then("the plan should remain in execute phase due to loop guard")
|
|
def step_verify_execute_blocked(context: Context) -> None:
|
|
"""Verify the plan is still in Execute phase (reversion was blocked)."""
|
|
assert context.result_plan.phase == PlanPhase.EXECUTE, (
|
|
f"Expected EXECUTE, got {context.result_plan.phase}"
|
|
)
|
|
assert context.result_plan.reversion_count == context.result_plan.MAX_REVERSIONS, (
|
|
f"Expected reversion_count={context.result_plan.MAX_REVERSIONS}, "
|
|
f"got {context.result_plan.reversion_count}"
|
|
)
|
|
|
|
|
|
# =================================================================
|
|
# Scenario: start_strategize resolves estimation actor names (P1-21)
|
|
# =================================================================
|
|
|
|
|
|
@given("a persisted plan lifecycle service with actor repository for coverage r2")
|
|
def step_create_persisted_service_with_actor_repo(context: Context) -> None:
|
|
"""Create PlanLifecycleService backed by a real UnitOfWork/ActorRepository."""
|
|
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as db_file:
|
|
db_path = db_file.name
|
|
db_url = f"sqlite:///{db_path}"
|
|
uow = UnitOfWork(database_url=db_url)
|
|
uow.init_database()
|
|
|
|
Settings._instance = None
|
|
settings = Settings()
|
|
context.service = PlanLifecycleService(settings=settings, unit_of_work=uow)
|
|
context.uow = uow
|
|
context.error = None
|
|
|
|
|
|
@given('a persisted estimation actor config named "{actor_name}"')
|
|
def step_create_persisted_estimation_actor_config(
|
|
context: Context, actor_name: str
|
|
) -> None:
|
|
"""Persist an actor config blob that preflight should resolve by name."""
|
|
config_blob = {
|
|
"name": actor_name,
|
|
"type": "llm",
|
|
"description": "Persisted estimator",
|
|
"model": "gpt-4",
|
|
"role_hint": "estimation",
|
|
"context_view": "strategist",
|
|
"response_format": {"type": "object"},
|
|
}
|
|
actor = Actor(
|
|
name=actor_name,
|
|
provider="local",
|
|
model="gpt-4",
|
|
config_blob=config_blob,
|
|
config_hash=Actor.compute_hash(config_blob),
|
|
)
|
|
with context.uow.transaction() as tx:
|
|
tx.actors.upsert(actor)
|
|
|
|
|
|
@given('an action "{name}" with estimation actor "{estimation_actor}" for coverage r2')
|
|
def step_create_named_action_with_estimation_actor(
|
|
context: Context, name: str, estimation_actor: str
|
|
) -> None:
|
|
"""Create an action configured with a namespaced estimation actor reference."""
|
|
context.action = _create_action_r2(
|
|
context,
|
|
name,
|
|
estimation_actor=estimation_actor,
|
|
)
|
|
|
|
|
|
@when("I start strategize with preflight capture enabled")
|
|
def step_start_strategize_with_preflight_capture(context: Context) -> None:
|
|
"""Capture actor_registry passed into preflight during start_strategize."""
|
|
original_run_all_checks = context.service.preflight_guardrail.run_all_checks
|
|
context.captured_preflight_kwargs = {}
|
|
|
|
def _capturing_run_all_checks(*args: object, **kwargs: object) -> object:
|
|
context.captured_preflight_kwargs = dict(kwargs)
|
|
return original_run_all_checks(*args, **kwargs)
|
|
|
|
context.service.preflight_guardrail.run_all_checks = _capturing_run_all_checks
|
|
|
|
pid = context.plan.identity.plan_id
|
|
context.plan = context.service.start_strategize(pid)
|
|
|
|
|
|
@then(
|
|
"preflight actor registry should include a resolved estimation actor config payload"
|
|
)
|
|
def step_verify_preflight_received_resolved_estimation_payload(
|
|
context: Context,
|
|
) -> None:
|
|
"""Verify estimation actor entry passed to preflight is resolved config dict."""
|
|
kwargs = getattr(context, "captured_preflight_kwargs", {})
|
|
actor_registry = kwargs.get("actor_registry")
|
|
assert isinstance(actor_registry, dict), (
|
|
f"Expected actor_registry dict, got {type(actor_registry).__name__}"
|
|
)
|
|
estimation = actor_registry.get("estimation")
|
|
assert isinstance(estimation, dict), (
|
|
f"Expected resolved estimation config dict, got {type(estimation).__name__}"
|
|
)
|
|
assert estimation.get("role_hint") == "estimation", (
|
|
f"Expected role_hint=estimation in resolved payload, got {estimation}"
|
|
)
|
|
assert isinstance(estimation.get("response_format"), dict), (
|
|
f"Expected response_format dict in resolved payload, got {estimation}"
|
|
)
|