test(e2e): workflow example 4 — multi-project dependency update (supervised profile) #815
+6
-1
@@ -489,7 +489,12 @@
|
||||
`EXECUTE_TYPES`. Code relying on `is_strategize_type` or `is_execute_type`
|
||||
returning `False` for `resource_selection` will see different results.
|
||||
Reclassification aligns with ADR-007 L72 and ADR-033 L74 which permit
|
||||
resource selection during planning. (#931)
|
||||
resource selection during planning. (#931)
|
||||
- Added E2E test for Workflow Example 4: Multi-Project Dependency Update
|
||||
(supervised profile). Exercises the full supervised plan lifecycle across 4
|
||||
git repositories (common-lib + 3 services), validates child plan spawning,
|
||||
dependency-ordered execution and apply, per-project validation attachment,
|
||||
and automation profile enforcement via the `agents plan use` CLI. (#750)
|
||||
- Added ResourceHandler CRUD and discovery methods: read, write, delete,
|
||||
list_children, diff, and discover_children. Frozen dataclass result types
|
||||
(Content, WriteResult, DeleteResult, DiffResult) added to the handler
|
||||
|
||||
@@ -0,0 +1,645 @@
|
||||
"""Step definitions for WF04 snapshot helper unit tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# The snapshot helper lives outside ``src/`` so we import via path manipulation.
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
_ROBOT_E2E = Path(__file__).resolve().parents[1].parent / "robot" / "e2e"
|
||||
if str(_ROBOT_E2E) not in sys.path:
|
||||
sys.path.append(str(_ROBOT_E2E))
|
||||
|
||||
from wf04_snapshot_helper import ( # noqa: E402
|
||||
_build_snapshot,
|
||||
_enum_value,
|
||||
_iso,
|
||||
count_decision_nodes,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _iso() steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a None timestamp value")
|
||||
def step_given_none_timestamp(context: Context) -> None:
|
||||
context.iso_input = None
|
||||
|
||||
|
||||
@given('a non-datetime truthy value "{value}"')
|
||||
def step_given_non_datetime_truthy(context: Context, value: str) -> None:
|
||||
context.iso_input = value
|
||||
|
||||
|
||||
@given("a naive datetime {iso_str}")
|
||||
def step_given_naive_datetime(context: Context, iso_str: str) -> None:
|
||||
context.iso_input = datetime.fromisoformat(iso_str)
|
||||
# Store for the "both" scenario
|
||||
if not hasattr(context, "iso_inputs"):
|
||||
context.iso_inputs = []
|
||||
context.iso_inputs.append(context.iso_input)
|
||||
|
||||
|
||||
@given("an aware datetime {iso_str} in timezone {tz_str}")
|
||||
def step_given_aware_datetime(context: Context, iso_str: str, tz_str: str) -> None:
|
||||
naive = datetime.fromisoformat(iso_str)
|
||||
if tz_str == "UTC":
|
||||
tz = UTC
|
||||
else:
|
||||
# Parse offset like "UTC+05:00"
|
||||
sign = 1 if "+" in tz_str else -1
|
||||
parts = tz_str.replace("UTC", "").lstrip("+-").split(":")
|
||||
hours = int(parts[0])
|
||||
minutes = int(parts[1]) if len(parts) > 1 else 0
|
||||
tz = timezone(timedelta(hours=sign * hours, minutes=sign * minutes))
|
||||
context.iso_input = naive.replace(tzinfo=tz)
|
||||
if not hasattr(context, "iso_inputs"):
|
||||
context.iso_inputs = []
|
||||
context.iso_inputs.append(context.iso_input)
|
||||
|
||||
|
||||
@when("_iso is called")
|
||||
def step_when_iso_called(context: Context) -> None:
|
||||
context.iso_result = _iso(context.iso_input)
|
||||
|
||||
|
||||
@when("_iso is called on both")
|
||||
def step_when_iso_called_on_both(context: Context) -> None:
|
||||
context.iso_results = [_iso(v) for v in context.iso_inputs]
|
||||
|
||||
|
||||
@then("the iso result should be an empty string")
|
||||
def step_then_result_empty(context: Context) -> None:
|
||||
assert context.iso_result == "", (
|
||||
f"Expected empty string, got {context.iso_result!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('the iso result should contain "{fragment}"')
|
||||
def step_then_result_contains(context: Context, fragment: str) -> None:
|
||||
assert fragment in context.iso_result, (
|
||||
f"Expected {fragment!r} in {context.iso_result!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("both iso results should be identical")
|
||||
def step_then_both_results_identical(context: Context) -> None:
|
||||
assert len(context.iso_results) == 2, "Expected exactly 2 results"
|
||||
assert context.iso_results[0] == context.iso_results[1], (
|
||||
f"Results differ: {context.iso_results[0]!r} vs {context.iso_results[1]!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _enum_value() steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeEnum(Enum):
|
||||
COMPLETED = "completed"
|
||||
|
||||
|
||||
@given('an enum-like object with value "{value}"')
|
||||
def step_given_enum_like(context: Context, value: str) -> None:
|
||||
context.enum_input = _FakeEnum(value)
|
||||
|
||||
|
||||
@given('a plain string "{value}"')
|
||||
def step_given_plain_string(context: Context, value: str) -> None:
|
||||
context.enum_input = value
|
||||
|
||||
|
||||
@when("_enum_value is called")
|
||||
def step_when_enum_value_called(context: Context) -> None:
|
||||
context.enum_result = _enum_value(context.enum_input)
|
||||
|
||||
|
||||
@then('the enum result should be "{expected}"')
|
||||
def step_then_result_equals(context: Context, expected: str) -> None:
|
||||
assert context.enum_result == expected, (
|
||||
f"Expected {expected!r}, got {context.enum_result!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# count_decision_nodes() steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("an empty list tree")
|
||||
def step_given_empty_tree(context: Context) -> None:
|
||||
empty: list[Any] = []
|
||||
context.tree_input = empty
|
||||
|
||||
|
||||
@given("a tree with one decision node")
|
||||
def step_given_one_decision_node(context: Context) -> None:
|
||||
context.tree_input = {"decision_id": "D001", "children": []}
|
||||
|
||||
|
||||
@given("a tree with a root and 2 child decision nodes")
|
||||
def step_given_nested_tree(context: Context) -> None:
|
||||
context.tree_input = {
|
||||
"decision_id": "D001",
|
||||
"children": [
|
||||
{"decision_id": "D002", "children": []},
|
||||
{"decision_id": "D003", "children": []},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@given("a tree with mixed decision and non-decision nodes")
|
||||
def step_given_mixed_tree(context: Context) -> None:
|
||||
context.tree_input = {
|
||||
"decision_id": "D001",
|
||||
"children": [
|
||||
{"decision_id": "", "children": []}, # no decision_id
|
||||
{"children": []}, # missing decision_id key
|
||||
{"decision_id": "D004", "children": []},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@given("a list of 2 independent decision trees")
|
||||
def step_given_list_of_trees(context: Context) -> None:
|
||||
context.tree_input = [
|
||||
{"decision_id": "D001", "children": []},
|
||||
{"decision_id": "D002", "children": []},
|
||||
]
|
||||
|
||||
|
||||
@given("a tree with depth {depth:d} and one decision node per level")
|
||||
def step_given_deep_tree(context: Context, depth: int) -> None:
|
||||
"""Build a linear chain of decision nodes at the given depth."""
|
||||
tree: dict[str, Any] = {"decision_id": f"D{depth:03d}", "children": []}
|
||||
current = tree
|
||||
for i in range(depth - 1, 0, -1):
|
||||
child: dict[str, Any] = {"decision_id": f"D{i:03d}", "children": []}
|
||||
current["children"] = [child]
|
||||
current = child
|
||||
context.tree_input = tree
|
||||
|
||||
|
||||
@when("count_decision_nodes is called with max_depth {max_depth:d}")
|
||||
def step_when_count_decision_nodes_with_depth(context: Context, max_depth: int) -> None:
|
||||
context.decision_count = count_decision_nodes(
|
||||
context.tree_input, max_depth=max_depth
|
||||
)
|
||||
|
||||
|
||||
@when("count_decision_nodes is called")
|
||||
def step_when_count_decision_nodes(context: Context) -> None:
|
||||
context.decision_count = count_decision_nodes(context.tree_input)
|
||||
|
||||
|
||||
@then("the decision node count should be {expected:d}")
|
||||
def step_then_count_equals(context: Context, expected: int) -> None:
|
||||
assert context.decision_count == expected, (
|
||||
f"Expected {expected}, got {context.decision_count}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_snapshot() steps (requires mocking the DI container)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_plan(
|
||||
plan_id: str = "01TEST00000000000000000001",
|
||||
*,
|
||||
subplan_statuses: list[Any] | None = None,
|
||||
multi_project_metadata: Any | None = None,
|
||||
validation_summary: dict[str, Any] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Create a mock Plan object for snapshot testing."""
|
||||
plan = MagicMock()
|
||||
plan.identity.plan_id = plan_id
|
||||
plan.subplan_statuses = subplan_statuses or []
|
||||
plan.multi_project_metadata = multi_project_metadata
|
||||
plan.validation_summary = validation_summary
|
||||
plan.phase = _FakeEnum.COMPLETED
|
||||
plan.processing_state = _FakeEnum.COMPLETED
|
||||
plan.timestamps.updated_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC)
|
||||
plan.timestamps.execute_started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC)
|
||||
plan.timestamps.execute_completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC)
|
||||
plan.timestamps.apply_started_at = datetime(2026, 3, 15, 11, 45, 0, tzinfo=UTC)
|
||||
plan.timestamps.applied_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC)
|
||||
return plan
|
||||
|
||||
|
||||
def _make_mock_subplan_status(
|
||||
subplan_id: str,
|
||||
target_resources: list[str] | None = None,
|
||||
) -> MagicMock:
|
||||
status = MagicMock()
|
||||
status.subplan_id = subplan_id
|
||||
status.status = _FakeEnum.COMPLETED
|
||||
status.started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC)
|
||||
status.completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC)
|
||||
status.target_resources = target_resources or []
|
||||
status.files_changed = 1
|
||||
return status
|
||||
|
||||
|
||||
def _make_mock_project_scope(project_name: str, resource_ids: list[str]) -> MagicMock:
|
||||
scope = MagicMock()
|
||||
scope.project_name = project_name
|
||||
scope.resource_ids = resource_ids
|
||||
return scope
|
||||
|
||||
|
||||
@given("a mocked lifecycle service that returns None for plan lookup")
|
||||
def step_given_mock_lifecycle_none(context: Context) -> None:
|
||||
context.mock_lifecycle = MagicMock()
|
||||
context.mock_lifecycle.get_plan.return_value = None
|
||||
context.mock_plan_id = "nonexistent"
|
||||
|
||||
|
||||
@given("a mocked lifecycle service with a plan that has no subplans")
|
||||
def step_given_mock_plan_no_subplans(context: Context) -> None:
|
||||
plan = _make_mock_plan()
|
||||
context.mock_lifecycle = MagicMock()
|
||||
context.mock_lifecycle.get_plan.return_value = plan
|
||||
context.mock_plan_id = plan.identity.plan_id
|
||||
|
||||
|
||||
@given("a mocked lifecycle service with a plan that has 2 subplans and project scopes")
|
||||
def step_given_mock_plan_with_subplans(context: Context) -> None:
|
||||
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
|
||||
scope2 = _make_mock_project_scope("proj-b", ["res-2"])
|
||||
metadata = MagicMock()
|
||||
metadata.project_scopes = [scope1, scope2]
|
||||
|
||||
status1 = _make_mock_subplan_status("SUB01", target_resources=["res-1"])
|
||||
status2 = _make_mock_subplan_status("SUB02", target_resources=["res-2"])
|
||||
|
||||
plan = _make_mock_plan(
|
||||
subplan_statuses=[status1, status2],
|
||||
multi_project_metadata=metadata,
|
||||
)
|
||||
|
||||
child1 = _make_mock_plan("SUB01")
|
||||
child1.validation_summary = {"required_passed": 1, "required_failed": 0}
|
||||
child2 = _make_mock_plan("SUB02")
|
||||
child2.validation_summary = {"required_passed": 1, "required_failed": 0}
|
||||
|
||||
lifecycle = MagicMock()
|
||||
lifecycle.get_plan.side_effect = lambda pid: {
|
||||
plan.identity.plan_id: plan,
|
||||
"SUB01": child1,
|
||||
"SUB02": child2,
|
||||
}.get(pid)
|
||||
|
||||
context.mock_lifecycle = lifecycle
|
||||
context.mock_plan_id = plan.identity.plan_id
|
||||
|
||||
|
||||
@given("a mocked lifecycle service with a subplan targeting an unmapped resource")
|
||||
def step_given_mock_plan_unmapped_resource(context: Context) -> None:
|
||||
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
|
||||
metadata = MagicMock()
|
||||
metadata.project_scopes = [scope1]
|
||||
|
||||
# Subplan targets res-1 (mapped) and res-unknown (unmapped)
|
||||
status1 = _make_mock_subplan_status(
|
||||
"SUB01", target_resources=["res-1", "res-unknown"]
|
||||
)
|
||||
plan = _make_mock_plan(
|
||||
subplan_statuses=[status1],
|
||||
multi_project_metadata=metadata,
|
||||
)
|
||||
|
||||
child1 = _make_mock_plan("SUB01")
|
||||
child1.validation_summary = None
|
||||
|
||||
lifecycle = MagicMock()
|
||||
lifecycle.get_plan.side_effect = lambda pid: {
|
||||
plan.identity.plan_id: plan,
|
||||
"SUB01": child1,
|
||||
}.get(pid)
|
||||
|
||||
context.mock_lifecycle = lifecycle
|
||||
context.mock_plan_id = plan.identity.plan_id
|
||||
|
||||
|
||||
@when('_build_snapshot is called with plan_id "{plan_id}"')
|
||||
def step_when_build_snapshot_with_id(context: Context, plan_id: str) -> None:
|
||||
mock_container = MagicMock()
|
||||
mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle
|
||||
|
||||
with patch("wf04_snapshot_helper.get_container", return_value=mock_container):
|
||||
try:
|
||||
context.snapshot_result = _build_snapshot(plan_id)
|
||||
context.snapshot_error = None
|
||||
except Exception as exc:
|
||||
context.snapshot_result = None
|
||||
context.snapshot_error = exc
|
||||
|
||||
|
||||
@when("_build_snapshot is called")
|
||||
def step_when_build_snapshot(context: Context) -> None:
|
||||
mock_container = MagicMock()
|
||||
mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle
|
||||
|
||||
with patch("wf04_snapshot_helper.get_container", return_value=mock_container):
|
||||
try:
|
||||
context.snapshot_result = _build_snapshot(context.mock_plan_id)
|
||||
context.snapshot_error = None
|
||||
except Exception as exc:
|
||||
context.snapshot_result = None
|
||||
context.snapshot_error = exc
|
||||
|
||||
|
||||
@then('the snapshot error should be ValueError containing "{fragment}"')
|
||||
def step_then_valueerror_raised(context: Context, fragment: str) -> None:
|
||||
assert context.snapshot_error is not None, (
|
||||
"Expected an exception but none was raised"
|
||||
)
|
||||
assert isinstance(context.snapshot_error, ValueError), (
|
||||
f"Expected ValueError, got {type(context.snapshot_error).__name__}"
|
||||
)
|
||||
assert fragment in str(context.snapshot_error), (
|
||||
f"Expected {fragment!r} in error message: {context.snapshot_error}"
|
||||
)
|
||||
|
||||
|
||||
@then("the snapshot should have subplan_count {expected:d}")
|
||||
def step_then_snapshot_subplan_count(context: Context, expected: int) -> None:
|
||||
assert context.snapshot_result is not None, "Snapshot should not be None"
|
||||
assert context.snapshot_result["subplan_count"] == expected, (
|
||||
f"Expected subplan_count={expected}, got {context.snapshot_result['subplan_count']}"
|
||||
)
|
||||
|
||||
|
||||
@then("the snapshot subplans list should be empty")
|
||||
def step_then_snapshot_subplans_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
assert context.snapshot_result["subplans"] == [], (
|
||||
f"Expected empty subplans, got {context.snapshot_result['subplans']}"
|
||||
)
|
||||
|
||||
|
||||
@then("each subplan should have mapped_projects populated")
|
||||
def step_then_subplan_mapped_projects(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
for sp in context.snapshot_result["subplans"]:
|
||||
assert "mapped_projects" in sp, "Subplan missing mapped_projects field"
|
||||
assert len(sp["mapped_projects"]) > 0, (
|
||||
f"Subplan {sp['subplan_id']} has empty mapped_projects"
|
||||
)
|
||||
|
||||
|
||||
@then("each subplan should have an unmapped_resources field")
|
||||
def step_then_subplan_unmapped_field(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
for sp in context.snapshot_result["subplans"]:
|
||||
assert "unmapped_resources" in sp, "Subplan missing unmapped_resources field"
|
||||
|
||||
|
||||
@then("the subplan unmapped_resources should contain the unknown resource ID")
|
||||
def step_then_subplan_unmapped_contains(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) == 1
|
||||
assert "res-unknown" in subplans[0]["unmapped_resources"], (
|
||||
f"Expected 'res-unknown' in unmapped_resources: {subplans[0]['unmapped_resources']}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Additional _build_snapshot() assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the snapshot plan_id should match the mocked plan ID")
|
||||
def step_then_snapshot_plan_id_matches(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
assert context.snapshot_result["plan_id"] == context.mock_plan_id, (
|
||||
f"Expected plan_id={context.mock_plan_id!r}, "
|
||||
f"got {context.snapshot_result['plan_id']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the snapshot project_scopes should be an empty list")
|
||||
def step_then_snapshot_project_scopes_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
assert context.snapshot_result["project_scopes"] == [], (
|
||||
f"Expected empty project_scopes, got {context.snapshot_result['project_scopes']}"
|
||||
)
|
||||
|
||||
|
||||
@then("the snapshot validation_summary should be None")
|
||||
def step_then_snapshot_validation_summary_none(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
assert context.snapshot_result["validation_summary"] is None, (
|
||||
f"Expected None validation_summary, "
|
||||
f"got {context.snapshot_result['validation_summary']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('subplan "{subplan_id}" should map to project "{project_name}"')
|
||||
def step_then_subplan_maps_to_project(
|
||||
context: Context, subplan_id: str, project_name: str
|
||||
) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
matching = [
|
||||
sp
|
||||
for sp in context.snapshot_result["subplans"]
|
||||
if sp["subplan_id"] == subplan_id
|
||||
]
|
||||
assert len(matching) == 1, (
|
||||
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
|
||||
)
|
||||
assert project_name in matching[0]["mapped_projects"], (
|
||||
f"Expected {project_name!r} in mapped_projects of {subplan_id}, "
|
||||
f"got {matching[0]['mapped_projects']}"
|
||||
)
|
||||
|
||||
|
||||
@then('the iso result should be exactly "{expected}"')
|
||||
def step_then_iso_result_exact(context: Context, expected: str) -> None:
|
||||
assert context.iso_result == expected, (
|
||||
f"Expected exactly {expected!r}, got {context.iso_result!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('subplan "{subplan_id}" status should be "{expected}"')
|
||||
def step_then_subplan_status(context: Context, subplan_id: str, expected: str) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
matching = [
|
||||
sp
|
||||
for sp in context.snapshot_result["subplans"]
|
||||
if sp["subplan_id"] == subplan_id
|
||||
]
|
||||
assert len(matching) == 1, (
|
||||
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
|
||||
)
|
||||
assert matching[0]["status"] == expected, (
|
||||
f"Expected status={expected!r} for {subplan_id}, got {matching[0]['status']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('subplan "{subplan_id}" child_phase should be "{expected}"')
|
||||
def step_then_subplan_child_phase(
|
||||
context: Context, subplan_id: str, expected: str
|
||||
) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
matching = [
|
||||
sp
|
||||
for sp in context.snapshot_result["subplans"]
|
||||
if sp["subplan_id"] == subplan_id
|
||||
]
|
||||
assert len(matching) == 1, (
|
||||
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
|
||||
)
|
||||
assert matching[0]["child_phase"] == expected, (
|
||||
f"Expected child_phase={expected!r} for {subplan_id}, "
|
||||
f"got {matching[0]['child_phase']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('subplan "{subplan_id}" started_at should be non-empty')
|
||||
def step_then_subplan_started_at_non_empty(context: Context, subplan_id: str) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
matching = [
|
||||
sp
|
||||
for sp in context.snapshot_result["subplans"]
|
||||
if sp["subplan_id"] == subplan_id
|
||||
]
|
||||
assert len(matching) == 1, (
|
||||
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
|
||||
)
|
||||
assert matching[0]["started_at"] != "", (
|
||||
f"Expected non-empty started_at for {subplan_id}, "
|
||||
f"got {matching[0]['started_at']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
'subplan "{subplan_id}" child_validation_summary required_passed should be {expected:d}'
|
||||
)
|
||||
def step_then_subplan_validation_required_passed(
|
||||
context: Context, subplan_id: str, expected: int
|
||||
) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
matching = [
|
||||
sp
|
||||
for sp in context.snapshot_result["subplans"]
|
||||
if sp["subplan_id"] == subplan_id
|
||||
]
|
||||
assert len(matching) == 1, (
|
||||
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
|
||||
)
|
||||
summary = matching[0].get("child_validation_summary")
|
||||
assert summary is not None, (
|
||||
f"Expected child_validation_summary for {subplan_id}, got None"
|
||||
)
|
||||
actual = int(summary.get("required_passed", 0))
|
||||
assert actual == expected, (
|
||||
f"Expected required_passed={expected} for {subplan_id}, got {actual}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_snapshot() child plan is None scenario
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a mocked lifecycle service with a subplan whose child plan is None")
|
||||
def step_given_mock_plan_child_none(context: Context) -> None:
|
||||
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
|
||||
metadata = MagicMock()
|
||||
metadata.project_scopes = [scope1]
|
||||
|
||||
status1 = _make_mock_subplan_status("SUB_NONE", target_resources=["res-1"])
|
||||
plan = _make_mock_plan(
|
||||
subplan_statuses=[status1],
|
||||
multi_project_metadata=metadata,
|
||||
)
|
||||
|
||||
lifecycle = MagicMock()
|
||||
# Parent plan found, but get_plan returns None for the child subplan
|
||||
lifecycle.get_plan.side_effect = lambda pid: (
|
||||
plan if pid == plan.identity.plan_id else None
|
||||
)
|
||||
|
||||
context.mock_lifecycle = lifecycle
|
||||
context.mock_plan_id = plan.identity.plan_id
|
||||
|
||||
|
||||
@then("the subplan child_phase should be an empty string")
|
||||
def step_then_child_phase_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["child_phase"] == "", (
|
||||
f"Expected empty child_phase, got {subplans[0]['child_phase']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan child_state should be an empty string")
|
||||
def step_then_child_state_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["child_state"] == "", (
|
||||
f"Expected empty child_state, got {subplans[0]['child_state']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan child_updated_at should be an empty string")
|
||||
def step_then_child_updated_at_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["child_updated_at"] == "", (
|
||||
f"Expected empty child_updated_at, got {subplans[0]['child_updated_at']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan execute_started_at should be an empty string")
|
||||
def step_then_execute_started_at_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["execute_started_at"] == "", (
|
||||
f"Expected empty execute_started_at, got {subplans[0]['execute_started_at']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan execute_completed_at should be an empty string")
|
||||
def step_then_execute_completed_at_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["execute_completed_at"] == "", (
|
||||
f"Expected empty execute_completed_at, got {subplans[0]['execute_completed_at']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan apply_started_at should be an empty string")
|
||||
def step_then_apply_started_at_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["apply_started_at"] == "", (
|
||||
f"Expected empty apply_started_at, got {subplans[0]['apply_started_at']!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan applied_at should be an empty string")
|
||||
def step_then_applied_at_empty(context: Context) -> None:
|
||||
assert context.snapshot_result is not None
|
||||
subplans = context.snapshot_result["subplans"]
|
||||
assert len(subplans) >= 1
|
||||
assert subplans[0]["applied_at"] == "", (
|
||||
f"Expected empty applied_at, got {subplans[0]['applied_at']!r}"
|
||||
)
|
||||
@@ -0,0 +1,141 @@
|
||||
@phase1 @e2e_helpers @wf04
|
||||
Feature: WF04 Snapshot Helper Utilities
|
||||
As a developer maintaining the WF04 E2E test infrastructure
|
||||
I want unit tests for the snapshot helper's utility functions
|
||||
So that timestamp normalisation, enum serialisation, decision counting,
|
||||
and snapshot building are verified without running a full E2E test
|
||||
|
||||
# --- _iso() timestamp normalisation ---
|
||||
|
||||
@iso_timestamp
|
||||
Scenario: _iso returns empty string for None
|
||||
Given a None timestamp value
|
||||
When _iso is called
|
||||
Then the iso result should be an empty string
|
||||
|
||||
@iso_timestamp
|
||||
Scenario: _iso returns empty string for non-datetime truthy value
|
||||
Given a non-datetime truthy value "not-a-date"
|
||||
When _iso is called
|
||||
Then the iso result should be an empty string
|
||||
|
||||
@iso_timestamp
|
||||
Scenario: _iso normalises naive datetime to UTC
|
||||
Given a naive datetime 2026-03-15T10:30:00
|
||||
When _iso is called
|
||||
Then the iso result should be exactly "2026-03-15T10:30:00+00:00"
|
||||
|
||||
@iso_timestamp
|
||||
Scenario: _iso normalises aware datetime to UTC
|
||||
Given an aware datetime 2026-03-15T10:30:00 in timezone UTC+05:00
|
||||
When _iso is called
|
||||
Then the iso result should be exactly "2026-03-15T05:30:00+00:00"
|
||||
|
||||
@iso_timestamp
|
||||
Scenario: _iso produces consistent format for mixed timezone inputs
|
||||
Given a naive datetime 2026-03-15T12:00:00
|
||||
And an aware datetime 2026-03-15T12:00:00 in timezone UTC
|
||||
When _iso is called on both
|
||||
Then both iso results should be identical
|
||||
|
||||
# --- _enum_value() serialisation ---
|
||||
|
||||
@enum_value
|
||||
Scenario: _enum_value returns .value for enum-like objects
|
||||
Given an enum-like object with value "completed"
|
||||
When _enum_value is called
|
||||
Then the enum result should be "completed"
|
||||
|
||||
@enum_value
|
||||
Scenario: _enum_value returns str for plain objects
|
||||
Given a plain string "hello"
|
||||
When _enum_value is called
|
||||
Then the enum result should be "hello"
|
||||
|
||||
# --- count_decision_nodes() ---
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes returns 0 for empty input
|
||||
Given an empty list tree
|
||||
When count_decision_nodes is called
|
||||
Then the decision node count should be 0
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes counts single root node
|
||||
Given a tree with one decision node
|
||||
When count_decision_nodes is called
|
||||
Then the decision node count should be 1
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes counts nested children
|
||||
Given a tree with a root and 2 child decision nodes
|
||||
When count_decision_nodes is called
|
||||
Then the decision node count should be 3
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes skips nodes without decision_id
|
||||
Given a tree with mixed decision and non-decision nodes
|
||||
When count_decision_nodes is called
|
||||
Then the decision node count should be 2
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes handles list input
|
||||
Given a list of 2 independent decision trees
|
||||
When count_decision_nodes is called
|
||||
Then the decision node count should be 2
|
||||
|
||||
@decision_count
|
||||
Scenario: count_decision_nodes truncates at max_depth
|
||||
Given a tree with depth 5 and one decision node per level
|
||||
When count_decision_nodes is called with max_depth 3
|
||||
Then the decision node count should be 3
|
||||
|
||||
# --- _build_snapshot() with mocked lifecycle service ---
|
||||
|
||||
@snapshot_build
|
||||
Scenario: _build_snapshot raises ValueError for missing plan
|
||||
Given a mocked lifecycle service that returns None for plan lookup
|
||||
When _build_snapshot is called with plan_id "nonexistent"
|
||||
Then the snapshot error should be ValueError containing "not found"
|
||||
|
||||
@snapshot_build
|
||||
Scenario: _build_snapshot returns correct structure for plan without subplans
|
||||
Given a mocked lifecycle service with a plan that has no subplans
|
||||
When _build_snapshot is called
|
||||
Then the snapshot should have subplan_count 0
|
||||
And the snapshot subplans list should be empty
|
||||
And the snapshot plan_id should match the mocked plan ID
|
||||
And the snapshot project_scopes should be an empty list
|
||||
And the snapshot validation_summary should be None
|
||||
|
||||
@snapshot_build
|
||||
Scenario: _build_snapshot maps resources to projects for subplans
|
||||
Given a mocked lifecycle service with a plan that has 2 subplans and project scopes
|
||||
When _build_snapshot is called
|
||||
Then the snapshot should have subplan_count 2
|
||||
And subplan "SUB01" should map to project "proj-a"
|
||||
And subplan "SUB02" should map to project "proj-b"
|
||||
And each subplan should have mapped_projects populated
|
||||
And each subplan should have an unmapped_resources field
|
||||
And subplan "SUB01" status should be "completed"
|
||||
And subplan "SUB01" child_phase should be "completed"
|
||||
And subplan "SUB01" started_at should be non-empty
|
||||
And subplan "SUB01" child_validation_summary required_passed should be 1
|
||||
|
||||
@snapshot_build
|
||||
Scenario: _build_snapshot includes unmapped_resources for unknown resource IDs
|
||||
Given a mocked lifecycle service with a subplan targeting an unmapped resource
|
||||
When _build_snapshot is called
|
||||
Then the subplan unmapped_resources should contain the unknown resource ID
|
||||
|
||||
@snapshot_build
|
||||
Scenario: _build_snapshot defaults child fields when child plan is None
|
||||
Given a mocked lifecycle service with a subplan whose child plan is None
|
||||
When _build_snapshot is called
|
||||
Then the subplan child_phase should be an empty string
|
||||
And the subplan child_state should be an empty string
|
||||
And the subplan child_updated_at should be an empty string
|
||||
And the subplan execute_started_at should be an empty string
|
||||
And the subplan execute_completed_at should be an empty string
|
||||
And the subplan apply_started_at should be an empty string
|
||||
And the subplan applied_at should be an empty string
|
||||
@@ -0,0 +1,577 @@
|
||||
*** Settings ***
|
||||
Documentation E2E test for Workflow Example 4: Multi-Project Dependency Update.
|
||||
...
|
||||
... Advanced scenario using the supervised automation profile.
|
||||
... Three microservices share a common library with a breaking
|
||||
... change (v1 to v2). CleverAgents creates a parent plan
|
||||
... targeting all 4 projects, spawns child subplans per project,
|
||||
... executes in dependency order (common-lib first, then services),
|
||||
... validates per project, and applies in dependency order.
|
||||
...
|
||||
... Zero mocking — real CLI, real LLM API keys.
|
||||
Resource common_e2e.resource
|
||||
Suite Setup WF04 Suite Setup
|
||||
Suite Teardown E2E Suite Teardown
|
||||
Force Tags E2E
|
||||
|
||||
*** Variables ***
|
||||
${ACTION_BASE} local/wf04-dep-update
|
||||
${LIB_BASE} wf04-common-lib
|
||||
${SVC1_BASE} wf04-svc-auth
|
||||
${SVC2_BASE} wf04-svc-billing
|
||||
${SVC3_BASE} wf04-svc-gateway
|
||||
${WF04_SNAPSHOT_HELPER} ${CURDIR}${/}wf04_snapshot_helper.py
|
||||
|
||||
*** Keywords ***
|
||||
WF04 Suite Setup
|
||||
[Documentation] E2E Suite Setup plus unique suffix generation and actor selection.
|
||||
E2E Suite Setup
|
||||
# Initialise the database so commands work in all tests.
|
||||
${init}= Run CleverAgents Command init --force --yes
|
||||
Should Be Equal As Integers ${init.rc} 0
|
||||
Should Not Contain ${init.stdout}${init.stderr} Traceback
|
||||
Should Not Contain ${init.stdout}${init.stderr} INTERNAL
|
||||
# Generate a unique suffix for resource/project names to avoid
|
||||
# UNIQUE constraint collisions on repeated or parallel CI runs.
|
||||
${suffix}= Evaluate __import__('uuid').uuid4().hex[:12]
|
||||
Set Suite Variable ${RUN_SUFFIX} ${suffix}
|
||||
# Derive unique names from base + suffix
|
||||
Set Suite Variable ${ACTION_NAME} ${ACTION_BASE}-${suffix}
|
||||
Set Suite Variable ${LIB_RESOURCE} ${LIB_BASE}-res-${suffix}
|
||||
Set Suite Variable ${SVC1_RESOURCE} ${SVC1_BASE}-res-${suffix}
|
||||
Set Suite Variable ${SVC2_RESOURCE} ${SVC2_BASE}-res-${suffix}
|
||||
Set Suite Variable ${SVC3_RESOURCE} ${SVC3_BASE}-res-${suffix}
|
||||
Set Suite Variable ${LIB_PROJECT} ${LIB_BASE}-proj-${suffix}
|
||||
Set Suite Variable ${SVC1_PROJECT} ${SVC1_BASE}-proj-${suffix}
|
||||
Set Suite Variable ${SVC2_PROJECT} ${SVC2_BASE}-proj-${suffix}
|
||||
Set Suite Variable ${SVC3_PROJECT} ${SVC3_BASE}-proj-${suffix}
|
||||
# Pick an actor that matches the available API key.
|
||||
${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', ''))
|
||||
${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
|
||||
IF ${has_openai}
|
||||
${actor}= Set Variable openai/gpt-4o
|
||||
ELSE IF ${has_anthropic}
|
||||
${actor}= Set Variable anthropic/claude-sonnet-4-20250514
|
||||
ELSE
|
||||
${actor}= Set Variable openai/gpt-4o
|
||||
END
|
||||
Set Suite Variable ${LLM_ACTOR} ${actor}
|
||||
|
||||
Create Library Repo
|
||||
[Documentation] Create temp git repo for the common library.
|
||||
${repo}= Create Temp Git Repo ${LIB_BASE}-${RUN_SUFFIX}
|
||||
Create Directory ${repo}${/}src
|
||||
${lib_content}= Catenate SEPARATOR=\n
|
||||
... """Common library v1 — shared utilities."""
|
||||
... ${EMPTY}
|
||||
... ${EMPTY}
|
||||
... __version__ = "1.0.0"
|
||||
... ${EMPTY}
|
||||
... ${EMPTY}
|
||||
... def connect(host, port):
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Connect to a service (v1 API)."""
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}return {"host": host, "port": port, "status": "connected"}
|
||||
Create File ${repo}${/}src${/}client.py ${lib_content}
|
||||
${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill
|
||||
Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr}
|
||||
${git_commit}= Run Process git commit -m Initial common library v1 cwd=${repo} timeout=30s on_timeout=kill
|
||||
Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr}
|
||||
RETURN ${repo}
|
||||
|
||||
Create Service Repo
|
||||
[Documentation] Create temp git repo for a microservice.
|
||||
[Arguments] ${name} ${import_line}
|
||||
${repo}= Create Temp Git Repo ${name}-${RUN_SUFFIX}
|
||||
Create Directory ${repo}${/}src
|
||||
${svc_content}= Catenate SEPARATOR=\n
|
||||
... """${name} service — depends on common-lib v1."""
|
||||
... ${import_line}
|
||||
... ${EMPTY}
|
||||
... ${EMPTY}
|
||||
... def start():
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}conn = connect("localhost", 8080)
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}return conn
|
||||
Create File ${repo}${/}src${/}app.py ${svc_content}
|
||||
Create File ${repo}${/}requirements.txt common-lib==1.0.0\n
|
||||
${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill
|
||||
Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr}
|
||||
${git_commit}= Run Process git commit -m Initial ${name} service cwd=${repo} timeout=30s on_timeout=kill
|
||||
Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr}
|
||||
RETURN ${repo}
|
||||
|
||||
Register Resource And Project
|
||||
[Documentation] Register a git-checkout resource and create a project.
|
||||
[Arguments] ${resource_name} ${project_name} ${repo_dir}
|
||||
${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} timeout=30s on_timeout=kill
|
||||
Should Be Equal As Integers ${branch_result.rc} 0 git rev-parse failed: ${branch_result.stderr}
|
||||
${branch}= Strip String ${branch_result.stdout}
|
||||
${r_res}= Run CleverAgents Command
|
||||
... resource add git-checkout ${resource_name}
|
||||
... --path ${repo_dir} --branch ${branch}
|
||||
Should Be Equal As Integers ${r_res.rc} 0 resource add failed: ${r_res.stderr}
|
||||
Should Not Contain ${r_res.stdout}${r_res.stderr} Traceback
|
||||
Should Not Contain ${r_res.stdout}${r_res.stderr} INTERNAL
|
||||
${r_proj}= Run CleverAgents Command
|
||||
... project create ${project_name}
|
||||
... --resource ${resource_name}
|
||||
Should Be Equal As Integers ${r_proj.rc} 0 project create failed: ${r_proj.stderr}
|
||||
Should Not Contain ${r_proj.stdout}${r_proj.stderr} Traceback
|
||||
Should Not Contain ${r_proj.stdout}${r_proj.stderr} INTERNAL
|
||||
|
||||
WF04 Test Teardown
|
||||
[Documentation] Log diagnostic context on failure for debugging.
|
||||
... Captures plan status and decision tree so CI failures
|
||||
... in this 25-minute LLM-dependent test have actionable data.
|
||||
${plan_id}= Get Variable Value ${WF04_PLAN_ID} ${EMPTY}
|
||||
IF '${plan_id}' != ''
|
||||
${status} ${result}= Run Keyword And Ignore Error
|
||||
... Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=30s
|
||||
IF '${status}' == 'PASS'
|
||||
Log Teardown plan status: ${result.stdout} WARN
|
||||
END
|
||||
${tree_status} ${tree_result}= Run Keyword And Ignore Error
|
||||
... Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=30s
|
||||
IF '${tree_status}' == 'PASS'
|
||||
Log Teardown plan tree: ${tree_result.stdout} WARN
|
||||
END
|
||||
END
|
||||
|
||||
Register Validation For Project
|
||||
[Documentation] Create a validation YAML, register it, and attach it to a project's resource.
|
||||
[Arguments] ${validation_name} ${resource_name} ${project_name}
|
||||
${val_yaml}= Catenate SEPARATOR=\n
|
||||
... name: ${validation_name}
|
||||
... description: "Simple pass-through validation for E2E testing"
|
||||
... source: custom
|
||||
... mode: required
|
||||
... code: |
|
||||
... ${SPACE}${SPACE}return {"passed": True, "data": {}, "message": "Validation passed"}
|
||||
... input_schema:
|
||||
... ${SPACE}${SPACE}type: object
|
||||
... ${SPACE}${SPACE}properties: {}
|
||||
... timeout: 30
|
||||
${val_path}= Set Variable ${SUITE_HOME}${/}${validation_name}.yaml
|
||||
Create File ${val_path} ${val_yaml}
|
||||
${r_add}= Run CleverAgents Command
|
||||
... validation add --config ${val_path} expected_rc=None
|
||||
Log Validation add rc=${r_add.rc} stdout=${r_add.stdout} stderr=${r_add.stderr}
|
||||
Should Be Equal As Integers ${r_add.rc} 0
|
||||
... validation add failed (rc=${r_add.rc}): ${r_add.stderr}
|
||||
Should Not Contain ${r_add.stdout}${r_add.stderr} Traceback
|
||||
Should Not Contain ${r_add.stdout}${r_add.stderr} INTERNAL
|
||||
# Positional args: <resource_name> <validation_name> (resource first, validation second)
|
||||
${r_attach}= Run CleverAgents Command
|
||||
... validation attach --project ${project_name}
|
||||
... ${resource_name} ${validation_name} expected_rc=None
|
||||
Log Validation attach rc=${r_attach.rc} stdout=${r_attach.stdout} stderr=${r_attach.stderr}
|
||||
Should Be Equal As Integers ${r_attach.rc} 0
|
||||
... validation attach failed for ${project_name} (rc=${r_attach.rc}): ${r_attach.stderr}
|
||||
Should Not Contain ${r_attach.stdout}${r_attach.stderr} Traceback
|
||||
Should Not Contain ${r_attach.stdout}${r_attach.stderr} INTERNAL
|
||||
|
||||
Attach Validation To Project
|
||||
[Documentation] Attach an already-registered validation to a project's resource.
|
||||
[Arguments] ${validation_name} ${resource_name} ${project_name}
|
||||
# Positional args: <resource_name> <validation_name> (resource first, validation second)
|
||||
${r_attach}= Run CleverAgents Command
|
||||
... validation attach --project ${project_name}
|
||||
... ${resource_name} ${validation_name} expected_rc=None
|
||||
Log Validation attach rc=${r_attach.rc} stdout=${r_attach.stdout} stderr=${r_attach.stderr}
|
||||
Should Be Equal As Integers ${r_attach.rc} 0
|
||||
... validation attach failed for ${project_name} (rc=${r_attach.rc}): ${r_attach.stderr}
|
||||
Should Not Contain ${r_attach.stdout}${r_attach.stderr} Traceback
|
||||
Should Not Contain ${r_attach.stdout}${r_attach.stderr} INTERNAL
|
||||
|
||||
Parse Json Payload
|
||||
[Documentation] Parse JSON object/array from stdout with optional log preamble.
|
||||
... Delegates to ``Extract JSON From Stdout`` which uses
|
||||
... ``json.JSONDecoder().raw_decode()`` for robustness against
|
||||
... trailing non-JSON output.
|
||||
[Arguments] ${stdout}
|
||||
${parsed}= Extract JSON From Stdout ${stdout}
|
||||
RETURN ${parsed}
|
||||
|
||||
Get WF04 Plan Snapshot
|
||||
[Documentation] Read parent/subplan metadata for deterministic WF04 assertions.
|
||||
[Arguments] ${plan_id}
|
||||
${snapshot_result}= Run Process
|
||||
... ${PYTHON} ${WF04_SNAPSHOT_HELPER} ${plan_id}
|
||||
... cwd=${SUITE_HOME} timeout=120s on_timeout=kill
|
||||
... env:CLEVERAGENTS_HOME=${SUITE_HOME}
|
||||
... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true
|
||||
... env:NO_COLOR=1
|
||||
... env:PYTHONPATH=${WORKSPACE}${/}src
|
||||
Should Be Equal As Integers ${snapshot_result.rc} 0
|
||||
... Snapshot helper failed (rc=${snapshot_result.rc}): ${snapshot_result.stderr}
|
||||
Should Not Be Empty ${snapshot_result.stdout}
|
||||
${snapshot}= Parse Json Payload ${snapshot_result.stdout}
|
||||
RETURN ${snapshot}
|
||||
|
||||
Verify WF04 Child Plan Spawning
|
||||
[Documentation] AC-4: verify exactly 4 child plans mapped one-per-project.
|
||||
[Arguments] ${snapshot}
|
||||
${subplans}= Evaluate $snapshot.get('subplans', [])
|
||||
${subplan_count}= Evaluate len($subplans)
|
||||
IF ${subplan_count} == 0
|
||||
Skip LLM produced no child plans — AC-4 child-plan mapping assertions were not exercised
|
||||
END
|
||||
Should Be Equal As Integers ${subplan_count} 4
|
||||
... Expected exactly 4 child plans (common-lib + 3 services), found ${subplan_count}
|
||||
${mapped_projects}= Evaluate sorted({p for sp in $subplans for p in sp.get('mapped_projects', []) if p})
|
||||
${expected_projects}= Create List ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT}
|
||||
${expected_sorted}= Evaluate sorted($expected_projects)
|
||||
${all_projects_covered}= Evaluate $mapped_projects == $expected_sorted
|
||||
Should Be True ${all_projects_covered}
|
||||
... Child plans should cover all 4 projects. expected=${expected_sorted} actual=${mapped_projects}
|
||||
${single_mapping}= Evaluate all(len(sp.get('mapped_projects', [])) == 1 for sp in $subplans)
|
||||
Should Be True ${single_mapping}
|
||||
... Each child plan should map to exactly one project scope
|
||||
|
||||
Verify WF04 Execution Order
|
||||
[Documentation] AC-5: common-lib executes before service subplans.
|
||||
[Arguments] ${snapshot}
|
||||
${subplans}= Evaluate $snapshot.get('subplans', [])
|
||||
${subplan_count}= Evaluate len($subplans)
|
||||
IF ${subplan_count} == 0
|
||||
Skip LLM produced no child plans — AC-5 execution-order assertions were not exercised
|
||||
END
|
||||
${lib_subplans}= Evaluate [sp for sp in $subplans if '${LIB_PROJECT}' in sp.get('mapped_projects', [])]
|
||||
${lib_count}= Evaluate len($lib_subplans)
|
||||
Should Be Equal As Integers ${lib_count} 1
|
||||
... Expected exactly one common-lib child plan, found ${lib_count}
|
||||
${lib_completed}= Evaluate $lib_subplans[0].get('completed_at') or $lib_subplans[0].get('execute_completed_at') or ''
|
||||
Should Not Be Empty ${lib_completed}
|
||||
... common-lib child plan completion timestamp is required for ordering checks
|
||||
${service_projects}= Create List ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT}
|
||||
${svc_subplans}= Evaluate [sp for sp in $subplans if any(p in $service_projects for p in sp.get('mapped_projects', []))]
|
||||
${svc_count}= Evaluate len($svc_subplans)
|
||||
Should Be Equal As Integers ${svc_count} 3
|
||||
... Expected exactly three service child plans, found ${svc_count}
|
||||
${svc_starts_present}= Evaluate all((sp.get('started_at') or sp.get('execute_started_at') or '') != '' for sp in $svc_subplans)
|
||||
Should Be True ${svc_starts_present}
|
||||
... Service child plans must expose start timestamps for execution-order checks
|
||||
${services_after_lib}= Evaluate all((sp.get('started_at') or sp.get('execute_started_at')) >= $lib_completed for sp in $svc_subplans)
|
||||
Should Be True ${services_after_lib}
|
||||
... Service execution must start only after common-lib execution completes
|
||||
|
||||
Verify WF04 Validation Outcomes
|
||||
[Documentation] AC-6: each child plan must report validation pass results.
|
||||
[Arguments] ${snapshot}
|
||||
${subplans}= Evaluate $snapshot.get('subplans', [])
|
||||
${subplan_count}= Evaluate len($subplans)
|
||||
IF ${subplan_count} == 0
|
||||
Skip LLM produced no child plans — AC-6 per-project validation assertions were not exercised
|
||||
END
|
||||
${validation_present}= Evaluate all(isinstance(sp.get('child_validation_summary'), dict) and len(sp.get('child_validation_summary')) > 0 for sp in $subplans)
|
||||
Should Be True ${validation_present}
|
||||
... Each child plan must expose a non-empty validation summary
|
||||
${validation_passed}= Evaluate all(int((sp.get('child_validation_summary') or {}).get('required_passed', 0) or 0) >= 1 and int((sp.get('child_validation_summary') or {}).get('required_failed', 0) or 0) == 0 for sp in $subplans)
|
||||
Should Be True ${validation_passed}
|
||||
... Each child plan must pass required validations (required_failed == 0)
|
||||
|
||||
Verify WF04 Apply Order
|
||||
[Documentation] AC-7: common-lib apply must complete before service applies.
|
||||
[Arguments] ${snapshot}
|
||||
${subplans}= Evaluate $snapshot.get('subplans', [])
|
||||
${subplan_count}= Evaluate len($subplans)
|
||||
IF ${subplan_count} == 0
|
||||
Skip LLM produced no child plans — AC-7 apply-order assertions were not exercised
|
||||
END
|
||||
${lib_subplans}= Evaluate [sp for sp in $subplans if '${LIB_PROJECT}' in sp.get('mapped_projects', [])]
|
||||
${lib_count}= Evaluate len($lib_subplans)
|
||||
Should Be Equal As Integers ${lib_count} 1
|
||||
... Expected exactly one common-lib child plan, found ${lib_count}
|
||||
${lib_applied}= Evaluate $lib_subplans[0].get('applied_at') or $lib_subplans[0].get('child_updated_at') or ''
|
||||
Should Not Be Empty ${lib_applied}
|
||||
... common-lib child plan apply timestamp is required for apply-order checks
|
||||
${service_projects}= Create List ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT}
|
||||
${svc_subplans}= Evaluate [sp for sp in $subplans if any(p in $service_projects for p in sp.get('mapped_projects', []))]
|
||||
${svc_count}= Evaluate len($svc_subplans)
|
||||
Should Be Equal As Integers ${svc_count} 3
|
||||
... Expected exactly three service child plans, found ${svc_count}
|
||||
${svc_apply_present}= Evaluate all((sp.get('applied_at') or sp.get('child_updated_at') or '') != '' for sp in $svc_subplans)
|
||||
Should Be True ${svc_apply_present}
|
||||
... Service child plans must expose apply/updated timestamps for apply-order checks
|
||||
${services_after_lib_apply}= Evaluate all((sp.get('applied_at') or sp.get('child_updated_at')) >= $lib_applied for sp in $svc_subplans)
|
||||
Should Be True ${services_after_lib_apply}
|
||||
... Service applies must occur after common-lib apply
|
||||
|
||||
Count Decision Nodes
|
||||
[Documentation] Recursively count decision nodes in a plan tree JSON structure.
|
||||
... Invokes ``wf04_snapshot_helper.py --count-nodes`` as a subprocess
|
||||
... to avoid importing the application DI container into the Robot
|
||||
... test runner process.
|
||||
[Arguments] ${tree_payload}
|
||||
${tree_json}= Evaluate __import__('json').dumps($tree_payload)
|
||||
${tmp_path}= Evaluate __import__('tempfile').NamedTemporaryFile(mode='w', suffix='.json', delete=False).name
|
||||
Evaluate __import__('pathlib').Path(r'${tmp_path}').write_text($tree_json, encoding='utf-8')
|
||||
${result}= Run Process
|
||||
... ${PYTHON} ${WF04_SNAPSHOT_HELPER} --count-nodes ${tmp_path}
|
||||
... timeout=30s on_timeout=kill
|
||||
Evaluate __import__('os').unlink(r'${tmp_path}')
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
... count-nodes failed (rc=${result.rc}): ${result.stderr}
|
||||
${count}= Convert To Integer ${result.stdout.strip()}
|
||||
RETURN ${count}
|
||||
|
||||
Verify Plan In Lifecycle List
|
||||
[Documentation] Verify a plan appears in lifecycle-list output.
|
||||
[Arguments] ${plan_id}
|
||||
${list_result}= Run CleverAgents Command
|
||||
... plan lifecycle-list --format json expected_rc=None timeout=120s
|
||||
Should Be Equal As Integers ${list_result.rc} 0
|
||||
... lifecycle-list failed (rc=${list_result.rc}): ${list_result.stderr}
|
||||
Output Should Contain ${list_result} ${plan_id}
|
||||
|
||||
*** Test Cases ***
|
||||
WF04 Multi Project Dependency Update Supervised Profile
|
||||
[Documentation] Full supervised-profile workflow: register 4 repos,
|
||||
... create multi-project action with invariants, plan use
|
||||
... targeting all 4 projects with --automation-profile supervised,
|
||||
... execute with child plan spawning in dependency order,
|
||||
... verify per-project validation, and apply in dependency order.
|
||||
[Timeout] 25 minutes
|
||||
[Teardown] WF04 Test Teardown
|
||||
Skip If No LLM Keys
|
||||
# Initialise test variable for teardown access.
|
||||
Set Test Variable ${WF04_PLAN_ID} ${EMPTY}
|
||||
|
||||
# ---- Create fixture repos ----
|
||||
${lib_repo}= Create Library Repo
|
||||
${svc1_repo}= Create Service Repo svc-auth from common_lib.client import connect
|
||||
${svc2_repo}= Create Service Repo svc-billing from common_lib.client import connect
|
||||
${svc3_repo}= Create Service Repo svc-gateway from common_lib.client import connect
|
||||
|
||||
# ---- Register resources and projects ----
|
||||
Register Resource And Project ${LIB_RESOURCE} ${LIB_PROJECT} ${lib_repo}
|
||||
Register Resource And Project ${SVC1_RESOURCE} ${SVC1_PROJECT} ${svc1_repo}
|
||||
Register Resource And Project ${SVC2_RESOURCE} ${SVC2_PROJECT} ${svc2_repo}
|
||||
Register Resource And Project ${SVC3_RESOURCE} ${SVC3_PROJECT} ${svc3_repo}
|
||||
|
||||
# ---- Register and attach validations for all 4 projects (AC-6) ----
|
||||
${val_name}= Set Variable local/wf04-val-${RUN_SUFFIX}
|
||||
Register Validation For Project ${val_name} ${LIB_RESOURCE} ${LIB_PROJECT}
|
||||
# Validation already registered; attach to remaining 3 projects
|
||||
Attach Validation To Project ${val_name} ${SVC1_RESOURCE} ${SVC1_PROJECT}
|
||||
Attach Validation To Project ${val_name} ${SVC2_RESOURCE} ${SVC2_PROJECT}
|
||||
Attach Validation To Project ${val_name} ${SVC3_RESOURCE} ${SVC3_PROJECT}
|
||||
|
||||
# ---- Create action with supervised profile and invariants ----
|
||||
${action_yaml}= Catenate SEPARATOR=\n
|
||||
... name: ${ACTION_NAME}
|
||||
... description: Update common-lib from v1 to v2 across all dependent services
|
||||
... definition_of_done: All services updated to use common-lib v2 API
|
||||
... strategy_actor: ${LLM_ACTOR}
|
||||
... execution_actor: ${LLM_ACTOR}
|
||||
... automation_profile: supervised
|
||||
... reusable: true
|
||||
... state: available
|
||||
... invariants:
|
||||
... ${SPACE}${SPACE}- "Each dependent project must be updated in its own child plan"
|
||||
... ${SPACE}${SPACE}- "All child plans must pass validation before any can be applied"
|
||||
... ${SPACE}${SPACE}- "The library update in common-lib must be applied first"
|
||||
${action_path}= Set Variable ${SUITE_HOME}${/}wf04_action.yaml
|
||||
Create File ${action_path} ${action_yaml}
|
||||
${r_action}= Run CleverAgents Command
|
||||
... action create --config ${action_path}
|
||||
Should Be Equal As Integers ${r_action.rc} 0
|
||||
... action create failed (rc=${r_action.rc}): ${r_action.stderr}
|
||||
Should Not Contain ${r_action.stdout}${r_action.stderr} Traceback
|
||||
Should Not Contain ${r_action.stdout}${r_action.stderr} INTERNAL
|
||||
Output Should Contain ${r_action} ${ACTION_NAME}
|
||||
|
||||
# ---- Plan use targeting ALL 4 projects with supervised profile (AC-3) ----
|
||||
${r_use}= Run CleverAgents Command
|
||||
... plan use ${ACTION_NAME}
|
||||
... ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT}
|
||||
... --automation-profile supervised
|
||||
... --format json
|
||||
... timeout=120s
|
||||
Should Be Equal As Integers ${r_use.rc} 0
|
||||
... plan use failed (rc=${r_use.rc}): ${r_use.stderr}
|
||||
Should Not Contain ${r_use.stdout}${r_use.stderr} Traceback
|
||||
Should Not Contain ${r_use.stdout}${r_use.stderr} INTERNAL
|
||||
${use_payload}= Parse Json Payload ${r_use.stdout}
|
||||
${plan_id}= Evaluate str($use_payload.get('plan_id', ''))
|
||||
Should Not Be Empty ${plan_id} msg=Expected plan_id in plan use JSON output
|
||||
Log Plan ID: ${plan_id}
|
||||
Set Test Variable ${WF04_PLAN_ID} ${plan_id}
|
||||
|
||||
# Verify plan targets all 4 projects exactly (AC-3)
|
||||
${use_projects}= Evaluate sorted([link.get('project_name', '') for link in $use_payload.get('project_links', []) if isinstance(link, dict)])
|
||||
${expected_projects}= Create List ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT}
|
||||
${expected_sorted}= Evaluate sorted($expected_projects)
|
||||
${projects_match}= Evaluate $use_projects == $expected_sorted
|
||||
Should Be True ${projects_match}
|
||||
... plan use should target all 4 projects. expected=${expected_sorted} actual=${use_projects}
|
||||
|
||||
# ---- Strategize ----
|
||||
# Supervised profile requires two explicit ``plan execute`` calls:
|
||||
# the first advances the plan through strategize, the second runs
|
||||
# actual execution. Both use the same CLI command.
|
||||
${r_strat}= Run CleverAgents Command
|
||||
... plan execute ${plan_id}
|
||||
... --format json expected_rc=None timeout=180s
|
||||
Log Strategize rc=${r_strat.rc} stdout=${r_strat.stdout} stderr=${r_strat.stderr}
|
||||
Should Not Contain ${r_strat.stdout}${r_strat.stderr} Traceback
|
||||
Should Not Contain ${r_strat.stdout}${r_strat.stderr} INTERNAL
|
||||
IF ${r_strat.rc} != 0
|
||||
Fail plan execute (strategize) failed (rc=${r_strat.rc}): ${r_strat.stderr}
|
||||
END
|
||||
|
||||
# ---- Decision tree — verify child plan spawning (AC-4) ----
|
||||
${r_tree}= Run CleverAgents Command
|
||||
... plan tree ${plan_id} --format json
|
||||
... expected_rc=None timeout=60s
|
||||
Should Be Equal As Integers ${r_tree.rc} 0
|
||||
... plan tree failed (rc=${r_tree.rc}): ${r_tree.stderr}
|
||||
Should Not Contain ${r_tree.stdout}${r_tree.stderr} Traceback
|
||||
Should Not Contain ${r_tree.stdout}${r_tree.stderr} INTERNAL
|
||||
Should Not Be Empty ${r_tree.stdout} Plan tree output should not be empty
|
||||
Log Decision tree: ${r_tree.stdout}
|
||||
|
||||
# Parse tree for child plan / decision structure
|
||||
${tree_payload}= Parse Json Payload ${r_tree.stdout}
|
||||
${decision_count}= Count Decision Nodes ${tree_payload}
|
||||
Log Decision tree contains ${decision_count} decision node(s)
|
||||
Should Be True ${decision_count} >= 1
|
||||
... Plan tree should contain at least one decision node after strategize (found ${decision_count})
|
||||
# Some providers/runs expose a minimal strategize tree before execute.
|
||||
# Child spawning is asserted deterministically after execute via snapshot.
|
||||
|
||||
# ---- Execute — dependency-ordered execution (AC-5) ----
|
||||
${r_exec}= Run CleverAgents Command
|
||||
... plan execute ${plan_id}
|
||||
... --format json expected_rc=None timeout=300s
|
||||
Log Execute rc=${r_exec.rc} stdout=${r_exec.stdout} stderr=${r_exec.stderr}
|
||||
Should Not Contain ${r_exec.stdout}${r_exec.stderr} Traceback
|
||||
Should Not Contain ${r_exec.stdout}${r_exec.stderr} INTERNAL
|
||||
IF ${r_exec.rc} != 0
|
||||
Fail plan execute failed (rc=${r_exec.rc}): ${r_exec.stderr}
|
||||
END
|
||||
|
||||
# Deterministic WF04 assertions after execute (AC-4/AC-5/AC-6)
|
||||
${exec_snapshot}= Get WF04 Plan Snapshot ${plan_id}
|
||||
# Guard: if the snapshot contains zero subplans, skip the entire test rather
|
||||
# than letting individual verification keywords silently skip all ACs.
|
||||
# This ensures CI reports show SKIPPED (visible) rather than PASSED (misleading).
|
||||
${exec_subplan_count}= Evaluate int($exec_snapshot.get('subplan_count', 0))
|
||||
IF ${exec_subplan_count} == 0
|
||||
Skip LLM produced 0 subplans — AC-4/5/6/7 verification cannot be exercised (entire test skipped)
|
||||
END
|
||||
Verify WF04 Child Plan Spawning ${exec_snapshot}
|
||||
Verify WF04 Execution Order ${exec_snapshot}
|
||||
Verify WF04 Validation Outcomes ${exec_snapshot}
|
||||
|
||||
# ---- Post-execute decision tree — verify child plan count (AC-4) ----
|
||||
${r_tree_post}= Run CleverAgents Command
|
||||
... plan tree ${plan_id} --format json
|
||||
... expected_rc=None timeout=60s
|
||||
Should Be Equal As Integers ${r_tree_post.rc} 0
|
||||
... plan tree (post-execute) failed (rc=${r_tree_post.rc}): ${r_tree_post.stderr}
|
||||
Should Not Contain ${r_tree_post.stdout}${r_tree_post.stderr} Traceback
|
||||
Should Not Contain ${r_tree_post.stdout}${r_tree_post.stderr} INTERNAL
|
||||
${tree_post_payload}= Parse Json Payload ${r_tree_post.stdout}
|
||||
${post_exec_decision_count}= Count Decision Nodes ${tree_post_payload}
|
||||
Log Post-execute decision tree contains ${post_exec_decision_count} decision node(s)
|
||||
# Require non-trivial tree depth after execute. Subplan spawning is
|
||||
# asserted deterministically via internal snapshot checks above.
|
||||
Should Be True ${post_exec_decision_count} >= 2
|
||||
... Plan tree should contain at least 2 decision nodes after execute (found ${post_exec_decision_count})
|
||||
# Execute should preserve or grow the tree — never shrink it.
|
||||
Should Be True ${post_exec_decision_count} >= ${decision_count}
|
||||
... Post-execute decision count (${post_exec_decision_count}) should not be less than post-strategize count (${decision_count})
|
||||
|
||||
# ---- AC-6 verified above via per-child validation summaries ----
|
||||
|
||||
# ---- Plan lifecycle-list — verify plan exists (replaces deprecated plan list) ----
|
||||
Verify Plan In Lifecycle List ${plan_id}
|
||||
|
||||
# ---- Verify plan status shows multi-project state ----
|
||||
${r_status_mid}= Run CleverAgents Command
|
||||
... plan status ${plan_id} --format json
|
||||
... expected_rc=None timeout=60s
|
||||
Should Be Equal As Integers ${r_status_mid.rc} 0
|
||||
... plan status failed (rc=${r_status_mid.rc}): ${r_status_mid.stderr}
|
||||
Should Not Contain ${r_status_mid.stdout}${r_status_mid.stderr} Traceback
|
||||
Should Not Contain ${r_status_mid.stdout}${r_status_mid.stderr} INTERNAL
|
||||
Output Should Contain ${r_status_mid} ${plan_id}
|
||||
# Check for automation profile reference in status
|
||||
${status_combined}= Set Variable ${r_status_mid.stdout}${r_status_mid.stderr}
|
||||
${status_lower}= Evaluate ($status_combined).lower()
|
||||
${has_profile_ref}= Evaluate 'supervised' in $status_lower or 'automation' in $status_lower or 'profile' in $status_lower
|
||||
Should Be True ${has_profile_ref}
|
||||
... Plan status should reference automation profile (supervised)
|
||||
# Parse automation_profile field for precise assertion
|
||||
${mid_profile}= Safe Parse Json Field ${r_status_mid.stdout} automation_profile
|
||||
Should Not Be Empty ${mid_profile}
|
||||
... automation_profile field should be present in plan status JSON
|
||||
Should Be Equal As Strings ${mid_profile} supervised
|
||||
... Plan automation profile should be supervised (found ${mid_profile})
|
||||
|
||||
# ---- Diff ----
|
||||
${r_diff}= Run CleverAgents Command
|
||||
... plan diff ${plan_id} --format plain
|
||||
... expected_rc=None timeout=60s
|
||||
Log Diff rc=${r_diff.rc} stdout=${r_diff.stdout} stderr=${r_diff.stderr}
|
||||
Should Not Contain ${r_diff.stdout}${r_diff.stderr} Traceback
|
||||
Should Not Contain ${r_diff.stdout}${r_diff.stderr} INTERNAL
|
||||
Should Be Equal As Integers ${r_diff.rc} 0
|
||||
... plan diff failed (rc=${r_diff.rc}): ${r_diff.stderr}
|
||||
|
||||
# ---- Apply — dependency-ordered apply (AC-7) ----
|
||||
${r_apply}= Run CleverAgents Command
|
||||
... plan lifecycle-apply ${plan_id} --yes --format json
|
||||
... expected_rc=None timeout=180s
|
||||
Log Apply rc=${r_apply.rc} stdout=${r_apply.stdout} stderr=${r_apply.stderr}
|
||||
Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback
|
||||
Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL
|
||||
IF ${r_apply.rc} == 0
|
||||
Output Should Contain ${r_apply} ${plan_id}
|
||||
# Verify the plan transitioned — check for apply-phase indicators (AC-7)
|
||||
${apply_phase}= Safe Parse Json Field ${r_apply.stdout} phase
|
||||
Should Not Be Empty ${apply_phase}
|
||||
... phase field should be present in lifecycle-apply JSON output
|
||||
${apply_phase_lower}= Evaluate ($apply_phase).lower()
|
||||
Should Contain ${apply_phase_lower} apply
|
||||
... Plan phase should indicate apply after lifecycle-apply (found ${apply_phase})
|
||||
${apply_snapshot}= Get WF04 Plan Snapshot ${plan_id}
|
||||
# Guard: apply snapshot must also contain subplans for AC-7 verification.
|
||||
# The exec guard above already skips the whole test if 0 subplans, so
|
||||
# reaching this point implies subplans existed post-execute. If they
|
||||
# disappeared post-apply, that would be a real regression worth flagging.
|
||||
${apply_subplan_count}= Evaluate int($apply_snapshot.get('subplan_count', 0))
|
||||
Should Be True ${apply_subplan_count} >= 1
|
||||
... Snapshot reported 0 subplans after apply — subplans existed post-execute but vanished post-apply
|
||||
Verify WF04 Apply Order ${apply_snapshot}
|
||||
Verify WF04 Validation Outcomes ${apply_snapshot}
|
||||
ELSE
|
||||
Fail lifecycle-apply failed (rc=${r_apply.rc}) stdout=${r_apply.stdout} stderr=${r_apply.stderr}
|
||||
END
|
||||
|
||||
# ---- Verify final status ----
|
||||
${r_status}= Run CleverAgents Command
|
||||
... plan status ${plan_id} --format json
|
||||
... expected_rc=None timeout=60s
|
||||
Should Be Equal As Integers ${r_status.rc} 0
|
||||
... plan status (final) failed (rc=${r_status.rc}): ${r_status.stderr}
|
||||
Should Not Contain ${r_status.stdout}${r_status.stderr} Traceback
|
||||
Should Not Contain ${r_status.stdout}${r_status.stderr} INTERNAL
|
||||
Should Not Be Empty ${r_status.stdout} Final plan status output should not be empty
|
||||
Output Should Contain ${r_status} ${plan_id}
|
||||
# Parse phase from final status and assert non-empty (proves lifecycle events processed)
|
||||
${final_phase}= Safe Parse Json Field ${r_status.stdout} phase
|
||||
${final_state}= Safe Parse Json Field ${r_status.stdout} processing_state
|
||||
Log Final phase=${final_phase} processing_state=${final_state}
|
||||
${final_state_populated}= Evaluate $final_phase != '' or $final_state != ''
|
||||
Should Be True ${final_state_populated}
|
||||
... Final plan status should have non-empty phase or processing_state
|
||||
# After a full lifecycle (execute + apply), phase or state should reflect completion
|
||||
${final_lower}= Evaluate ($final_phase).lower() if $final_phase else ''
|
||||
${state_lower}= Evaluate ($final_state).lower() if $final_state else ''
|
||||
${is_terminal}= Evaluate 'apply' in $final_lower or 'complete' in $final_lower or 'done' in $final_lower or 'complete' in $state_lower or 'done' in $state_lower or 'applied' in $state_lower
|
||||
Should Be True ${is_terminal}
|
||||
... Final status should indicate a terminal/applied state (phase=${final_phase}, state=${final_state})
|
||||
|
||||
# ---- Final: plan should still appear in lifecycle-list ----
|
||||
Verify Plan In Lifecycle List ${plan_id}
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Snapshot helper for WF04 E2E assertions.
|
||||
|
||||
This script is executed by ``wf04_multi_project.robot`` to collect
|
||||
deterministic parent/subplan metadata from the lifecycle service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SRC = ROOT / "src"
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.append(str(SRC))
|
||||
|
||||
from cleveragents.application.container import get_container # noqa: E402
|
||||
|
||||
|
||||
def _iso(value: Any) -> str:
|
||||
"""Return ISO timestamp string normalised to UTC, or empty string.
|
||||
|
||||
Ensures all timestamps are in a consistent UTC-aware format so that
|
||||
lexicographic string comparison in Robot assertions is reliable.
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
if not isinstance(value, datetime):
|
||||
return ""
|
||||
# Normalise naive datetimes by assuming UTC
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
# Convert to UTC and produce a consistent representation
|
||||
return value.astimezone(UTC).isoformat()
|
||||
|
||||
|
||||
def _enum_value(value: Any) -> str:
|
||||
"""Return enum ``.value`` when available, else ``str(value)``."""
|
||||
return value.value if hasattr(value, "value") else str(value)
|
||||
|
||||
|
||||
DecisionTree = dict[str, Any] | list[Any]
|
||||
|
||||
|
||||
def count_decision_nodes(root: DecisionTree | Any, *, max_depth: int = 50) -> int:
|
||||
"""Recursively count decision nodes in a plan tree structure.
|
||||
|
||||
Walks the tree rooted at *root* and counts every ``dict`` node whose
|
||||
``decision_id`` field is non-empty. Works for both dict (single root)
|
||||
and list (forest) inputs.
|
||||
|
||||
A *max_depth* guard prevents unbounded recursion on malformed trees.
|
||||
"""
|
||||
if max_depth <= 0:
|
||||
return 0
|
||||
if isinstance(root, list):
|
||||
return sum(count_decision_nodes(item, max_depth=max_depth) for item in root)
|
||||
if isinstance(root, dict):
|
||||
has_decision = 1 if str(root.get("decision_id", "")).strip() else 0
|
||||
children = root.get("children") or []
|
||||
return has_decision + sum(
|
||||
count_decision_nodes(child, max_depth=max_depth - 1) for child in children
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _build_snapshot(plan_id: str) -> dict[str, Any]:
|
||||
"""Build a serialisable snapshot for WF04 verification assertions."""
|
||||
lifecycle_service = get_container().plan_lifecycle_service()
|
||||
plan = lifecycle_service.get_plan(plan_id)
|
||||
if plan is None:
|
||||
msg = f"plan '{plan_id}' not found"
|
||||
raise ValueError(msg)
|
||||
|
||||
project_scopes: list[dict[str, Any]] = []
|
||||
resource_to_project: dict[str, str] = {}
|
||||
|
||||
if plan.multi_project_metadata is not None:
|
||||
for scope in plan.multi_project_metadata.project_scopes:
|
||||
project_scopes.append(
|
||||
{
|
||||
"project_name": scope.project_name,
|
||||
"resource_ids": list(scope.resource_ids),
|
||||
}
|
||||
)
|
||||
for resource_id in scope.resource_ids:
|
||||
resource_to_project[resource_id] = scope.project_name
|
||||
|
||||
subplans: list[dict[str, Any]] = []
|
||||
for status in plan.subplan_statuses:
|
||||
child_plan = lifecycle_service.get_plan(status.subplan_id)
|
||||
mapped_projects = sorted(
|
||||
{
|
||||
resource_to_project[resource_id]
|
||||
for resource_id in status.target_resources
|
||||
if resource_id in resource_to_project
|
||||
}
|
||||
)
|
||||
unmapped_resources = sorted(
|
||||
{
|
||||
resource_id
|
||||
for resource_id in status.target_resources
|
||||
if resource_id not in resource_to_project
|
||||
}
|
||||
)
|
||||
|
||||
subplans.append(
|
||||
{
|
||||
"subplan_id": status.subplan_id,
|
||||
"status": _enum_value(status.status),
|
||||
"started_at": _iso(status.started_at),
|
||||
"completed_at": _iso(status.completed_at),
|
||||
"target_resources": list(status.target_resources),
|
||||
"mapped_projects": mapped_projects,
|
||||
"unmapped_resources": unmapped_resources,
|
||||
"files_changed": status.files_changed,
|
||||
"child_phase": (
|
||||
_enum_value(child_plan.phase) if child_plan is not None else ""
|
||||
),
|
||||
"child_state": (
|
||||
_enum_value(child_plan.processing_state)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"child_updated_at": (
|
||||
_iso(child_plan.timestamps.updated_at)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"execute_started_at": (
|
||||
_iso(child_plan.timestamps.execute_started_at)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"execute_completed_at": (
|
||||
_iso(child_plan.timestamps.execute_completed_at)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"apply_started_at": (
|
||||
_iso(child_plan.timestamps.apply_started_at)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"applied_at": (
|
||||
_iso(child_plan.timestamps.applied_at)
|
||||
if child_plan is not None
|
||||
else ""
|
||||
),
|
||||
"child_validation_summary": (
|
||||
child_plan.validation_summary if child_plan is not None else None
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"plan_id": plan.identity.plan_id,
|
||||
"subplan_count": len(plan.subplan_statuses),
|
||||
"validation_summary": plan.validation_summary,
|
||||
"project_scopes": project_scopes,
|
||||
"subplans": subplans,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entrypoint.
|
||||
|
||||
Modes:
|
||||
wf04_snapshot_helper.py <plan_id>
|
||||
Build and print the full snapshot JSON for a plan.
|
||||
wf04_snapshot_helper.py --count-nodes <json_file>
|
||||
Read a JSON tree from *json_file* (or ``-`` for stdin) and
|
||||
print the decision-node count to stdout.
|
||||
"""
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "--count-nodes":
|
||||
return _cli_count_nodes(sys.argv[2])
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print(
|
||||
"Usage: wf04_snapshot_helper.py <plan_id>\n"
|
||||
" wf04_snapshot_helper.py --count-nodes <json_file|->\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
plan_id = sys.argv[1]
|
||||
try:
|
||||
snapshot = _build_snapshot(plan_id)
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
# Intentional catch-all: CLI entrypoint must never propagate
|
||||
# unhandled exceptions — report full traceback and exit non-zero.
|
||||
except Exception as exc:
|
||||
print(f"{exc}\n{traceback.format_exc()}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(json.dumps(snapshot, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
def _cli_count_nodes(json_source: str) -> int:
|
||||
"""Read a JSON tree and print the decision-node count."""
|
||||
try:
|
||||
if json_source == "-":
|
||||
tree = json.load(sys.stdin)
|
||||
else:
|
||||
tree = json.loads(Path(json_source).read_text(encoding="utf-8"))
|
||||
print(count_decision_nodes(tree))
|
||||
except (ValueError, OSError, json.JSONDecodeError) as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
# Intentional catch-all: CLI must not propagate exceptions.
|
||||
except Exception as exc:
|
||||
print(f"{exc}\n{traceback.format_exc()}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user