From 845cf61b47a089a7190db384133a62da92f2f4b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Fri, 13 Mar 2026 16:48:27 +0000 Subject: [PATCH] =?UTF-8?q?test(e2e):=20workflow=20example=204=20=E2=80=94?= =?UTF-8?q?=20multi-project=20dependency=20update=20(supervised=20profile)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement WF04 E2E test coverage for multi-project dependency update using the supervised automation profile. A parent plan targets four projects (common-lib + 3 services), spawns child subplans per project, executes in dependency order, validates per project, and applies in dependency order. Key components: - Robot test (robot/e2e/wf04_multi_project.robot): full supervised workflow exercising plan use, strategize, execute, validate, apply, and final status. JSON output parsing via raw_decode for resilience. Test-level guard skips the entire test (visible SKIPPED in CI) when the LLM produces zero subplans, preventing silent AC-4/5/6/7 bypass. Count Decision Nodes keyword delegates to the Python helper to avoid duplicating recursive tree-walking logic. - Snapshot helper (robot/e2e/wf04_snapshot_helper.py): collects deterministic parent/subplan metadata from the lifecycle service. Timestamps normalised to UTC. Depth-guarded count_decision_nodes() function (max_depth=50) for safe reuse from Robot keywords. sys.path.append (not insert) to avoid shadowing the standard library. - Behave unit tests (features/wf04_snapshot_helper.feature + steps): 17 scenarios covering _iso(), _enum_value(), count_decision_nodes(), and _build_snapshot() with mocked lifecycle service. Includes tests for no-subplans (verifies plan_id, project_scopes, validation_summary), with-subplans (verifies concrete mapped project names and subplan_count), child-plan-is-None (verifies empty-string defaults), unmapped resources, and exact-match aware-datetime UTC normalisation. ISSUES CLOSED: #750 --- CHANGELOG.md | 7 +- features/steps/wf04_snapshot_helper_steps.py | 645 +++++++++++++++++++ features/wf04_snapshot_helper.feature | 141 ++++ robot/e2e/wf04_multi_project.robot | 577 +++++++++++++++++ robot/e2e/wf04_snapshot_helper.py | 225 +++++++ 5 files changed, 1594 insertions(+), 1 deletion(-) create mode 100644 features/steps/wf04_snapshot_helper_steps.py create mode 100644 features/wf04_snapshot_helper.feature create mode 100644 robot/e2e/wf04_multi_project.robot create mode 100644 robot/e2e/wf04_snapshot_helper.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 764dbe542..31e62213f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -489,7 +489,12 @@ `EXECUTE_TYPES`. Code relying on `is_strategize_type` or `is_execute_type` returning `False` for `resource_selection` will see different results. Reclassification aligns with ADR-007 L72 and ADR-033 L74 which permit - resource selection during planning. (#931) + resource selection during planning. (#931) +- Added E2E test for Workflow Example 4: Multi-Project Dependency Update + (supervised profile). Exercises the full supervised plan lifecycle across 4 + git repositories (common-lib + 3 services), validates child plan spawning, + dependency-ordered execution and apply, per-project validation attachment, + and automation profile enforcement via the `agents plan use` CLI. (#750) - Added ResourceHandler CRUD and discovery methods: read, write, delete, list_children, diff, and discover_children. Frozen dataclass result types (Content, WriteResult, DeleteResult, DiffResult) added to the handler diff --git a/features/steps/wf04_snapshot_helper_steps.py b/features/steps/wf04_snapshot_helper_steps.py new file mode 100644 index 000000000..2a5c75b07 --- /dev/null +++ b/features/steps/wf04_snapshot_helper_steps.py @@ -0,0 +1,645 @@ +"""Step definitions for WF04 snapshot helper unit tests.""" + +from __future__ import annotations + +# The snapshot helper lives outside ``src/`` so we import via path manipulation. +import sys +from datetime import UTC, datetime, timedelta, timezone +from enum import Enum +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context + +_ROBOT_E2E = Path(__file__).resolve().parents[1].parent / "robot" / "e2e" +if str(_ROBOT_E2E) not in sys.path: + sys.path.append(str(_ROBOT_E2E)) + +from wf04_snapshot_helper import ( # noqa: E402 + _build_snapshot, + _enum_value, + _iso, + count_decision_nodes, +) + +# --------------------------------------------------------------------------- +# _iso() steps +# --------------------------------------------------------------------------- + + +@given("a None timestamp value") +def step_given_none_timestamp(context: Context) -> None: + context.iso_input = None + + +@given('a non-datetime truthy value "{value}"') +def step_given_non_datetime_truthy(context: Context, value: str) -> None: + context.iso_input = value + + +@given("a naive datetime {iso_str}") +def step_given_naive_datetime(context: Context, iso_str: str) -> None: + context.iso_input = datetime.fromisoformat(iso_str) + # Store for the "both" scenario + if not hasattr(context, "iso_inputs"): + context.iso_inputs = [] + context.iso_inputs.append(context.iso_input) + + +@given("an aware datetime {iso_str} in timezone {tz_str}") +def step_given_aware_datetime(context: Context, iso_str: str, tz_str: str) -> None: + naive = datetime.fromisoformat(iso_str) + if tz_str == "UTC": + tz = UTC + else: + # Parse offset like "UTC+05:00" + sign = 1 if "+" in tz_str else -1 + parts = tz_str.replace("UTC", "").lstrip("+-").split(":") + hours = int(parts[0]) + minutes = int(parts[1]) if len(parts) > 1 else 0 + tz = timezone(timedelta(hours=sign * hours, minutes=sign * minutes)) + context.iso_input = naive.replace(tzinfo=tz) + if not hasattr(context, "iso_inputs"): + context.iso_inputs = [] + context.iso_inputs.append(context.iso_input) + + +@when("_iso is called") +def step_when_iso_called(context: Context) -> None: + context.iso_result = _iso(context.iso_input) + + +@when("_iso is called on both") +def step_when_iso_called_on_both(context: Context) -> None: + context.iso_results = [_iso(v) for v in context.iso_inputs] + + +@then("the iso result should be an empty string") +def step_then_result_empty(context: Context) -> None: + assert context.iso_result == "", ( + f"Expected empty string, got {context.iso_result!r}" + ) + + +@then('the iso result should contain "{fragment}"') +def step_then_result_contains(context: Context, fragment: str) -> None: + assert fragment in context.iso_result, ( + f"Expected {fragment!r} in {context.iso_result!r}" + ) + + +@then("both iso results should be identical") +def step_then_both_results_identical(context: Context) -> None: + assert len(context.iso_results) == 2, "Expected exactly 2 results" + assert context.iso_results[0] == context.iso_results[1], ( + f"Results differ: {context.iso_results[0]!r} vs {context.iso_results[1]!r}" + ) + + +# --------------------------------------------------------------------------- +# _enum_value() steps +# --------------------------------------------------------------------------- + + +class _FakeEnum(Enum): + COMPLETED = "completed" + + +@given('an enum-like object with value "{value}"') +def step_given_enum_like(context: Context, value: str) -> None: + context.enum_input = _FakeEnum(value) + + +@given('a plain string "{value}"') +def step_given_plain_string(context: Context, value: str) -> None: + context.enum_input = value + + +@when("_enum_value is called") +def step_when_enum_value_called(context: Context) -> None: + context.enum_result = _enum_value(context.enum_input) + + +@then('the enum result should be "{expected}"') +def step_then_result_equals(context: Context, expected: str) -> None: + assert context.enum_result == expected, ( + f"Expected {expected!r}, got {context.enum_result!r}" + ) + + +# --------------------------------------------------------------------------- +# count_decision_nodes() steps +# --------------------------------------------------------------------------- + + +@given("an empty list tree") +def step_given_empty_tree(context: Context) -> None: + empty: list[Any] = [] + context.tree_input = empty + + +@given("a tree with one decision node") +def step_given_one_decision_node(context: Context) -> None: + context.tree_input = {"decision_id": "D001", "children": []} + + +@given("a tree with a root and 2 child decision nodes") +def step_given_nested_tree(context: Context) -> None: + context.tree_input = { + "decision_id": "D001", + "children": [ + {"decision_id": "D002", "children": []}, + {"decision_id": "D003", "children": []}, + ], + } + + +@given("a tree with mixed decision and non-decision nodes") +def step_given_mixed_tree(context: Context) -> None: + context.tree_input = { + "decision_id": "D001", + "children": [ + {"decision_id": "", "children": []}, # no decision_id + {"children": []}, # missing decision_id key + {"decision_id": "D004", "children": []}, + ], + } + + +@given("a list of 2 independent decision trees") +def step_given_list_of_trees(context: Context) -> None: + context.tree_input = [ + {"decision_id": "D001", "children": []}, + {"decision_id": "D002", "children": []}, + ] + + +@given("a tree with depth {depth:d} and one decision node per level") +def step_given_deep_tree(context: Context, depth: int) -> None: + """Build a linear chain of decision nodes at the given depth.""" + tree: dict[str, Any] = {"decision_id": f"D{depth:03d}", "children": []} + current = tree + for i in range(depth - 1, 0, -1): + child: dict[str, Any] = {"decision_id": f"D{i:03d}", "children": []} + current["children"] = [child] + current = child + context.tree_input = tree + + +@when("count_decision_nodes is called with max_depth {max_depth:d}") +def step_when_count_decision_nodes_with_depth(context: Context, max_depth: int) -> None: + context.decision_count = count_decision_nodes( + context.tree_input, max_depth=max_depth + ) + + +@when("count_decision_nodes is called") +def step_when_count_decision_nodes(context: Context) -> None: + context.decision_count = count_decision_nodes(context.tree_input) + + +@then("the decision node count should be {expected:d}") +def step_then_count_equals(context: Context, expected: int) -> None: + assert context.decision_count == expected, ( + f"Expected {expected}, got {context.decision_count}" + ) + + +# --------------------------------------------------------------------------- +# _build_snapshot() steps (requires mocking the DI container) +# --------------------------------------------------------------------------- + + +def _make_mock_plan( + plan_id: str = "01TEST00000000000000000001", + *, + subplan_statuses: list[Any] | None = None, + multi_project_metadata: Any | None = None, + validation_summary: dict[str, Any] | None = None, +) -> MagicMock: + """Create a mock Plan object for snapshot testing.""" + plan = MagicMock() + plan.identity.plan_id = plan_id + plan.subplan_statuses = subplan_statuses or [] + plan.multi_project_metadata = multi_project_metadata + plan.validation_summary = validation_summary + plan.phase = _FakeEnum.COMPLETED + plan.processing_state = _FakeEnum.COMPLETED + plan.timestamps.updated_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC) + plan.timestamps.execute_started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC) + plan.timestamps.execute_completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC) + plan.timestamps.apply_started_at = datetime(2026, 3, 15, 11, 45, 0, tzinfo=UTC) + plan.timestamps.applied_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC) + return plan + + +def _make_mock_subplan_status( + subplan_id: str, + target_resources: list[str] | None = None, +) -> MagicMock: + status = MagicMock() + status.subplan_id = subplan_id + status.status = _FakeEnum.COMPLETED + status.started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC) + status.completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC) + status.target_resources = target_resources or [] + status.files_changed = 1 + return status + + +def _make_mock_project_scope(project_name: str, resource_ids: list[str]) -> MagicMock: + scope = MagicMock() + scope.project_name = project_name + scope.resource_ids = resource_ids + return scope + + +@given("a mocked lifecycle service that returns None for plan lookup") +def step_given_mock_lifecycle_none(context: Context) -> None: + context.mock_lifecycle = MagicMock() + context.mock_lifecycle.get_plan.return_value = None + context.mock_plan_id = "nonexistent" + + +@given("a mocked lifecycle service with a plan that has no subplans") +def step_given_mock_plan_no_subplans(context: Context) -> None: + plan = _make_mock_plan() + context.mock_lifecycle = MagicMock() + context.mock_lifecycle.get_plan.return_value = plan + context.mock_plan_id = plan.identity.plan_id + + +@given("a mocked lifecycle service with a plan that has 2 subplans and project scopes") +def step_given_mock_plan_with_subplans(context: Context) -> None: + scope1 = _make_mock_project_scope("proj-a", ["res-1"]) + scope2 = _make_mock_project_scope("proj-b", ["res-2"]) + metadata = MagicMock() + metadata.project_scopes = [scope1, scope2] + + status1 = _make_mock_subplan_status("SUB01", target_resources=["res-1"]) + status2 = _make_mock_subplan_status("SUB02", target_resources=["res-2"]) + + plan = _make_mock_plan( + subplan_statuses=[status1, status2], + multi_project_metadata=metadata, + ) + + child1 = _make_mock_plan("SUB01") + child1.validation_summary = {"required_passed": 1, "required_failed": 0} + child2 = _make_mock_plan("SUB02") + child2.validation_summary = {"required_passed": 1, "required_failed": 0} + + lifecycle = MagicMock() + lifecycle.get_plan.side_effect = lambda pid: { + plan.identity.plan_id: plan, + "SUB01": child1, + "SUB02": child2, + }.get(pid) + + context.mock_lifecycle = lifecycle + context.mock_plan_id = plan.identity.plan_id + + +@given("a mocked lifecycle service with a subplan targeting an unmapped resource") +def step_given_mock_plan_unmapped_resource(context: Context) -> None: + scope1 = _make_mock_project_scope("proj-a", ["res-1"]) + metadata = MagicMock() + metadata.project_scopes = [scope1] + + # Subplan targets res-1 (mapped) and res-unknown (unmapped) + status1 = _make_mock_subplan_status( + "SUB01", target_resources=["res-1", "res-unknown"] + ) + plan = _make_mock_plan( + subplan_statuses=[status1], + multi_project_metadata=metadata, + ) + + child1 = _make_mock_plan("SUB01") + child1.validation_summary = None + + lifecycle = MagicMock() + lifecycle.get_plan.side_effect = lambda pid: { + plan.identity.plan_id: plan, + "SUB01": child1, + }.get(pid) + + context.mock_lifecycle = lifecycle + context.mock_plan_id = plan.identity.plan_id + + +@when('_build_snapshot is called with plan_id "{plan_id}"') +def step_when_build_snapshot_with_id(context: Context, plan_id: str) -> None: + mock_container = MagicMock() + mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle + + with patch("wf04_snapshot_helper.get_container", return_value=mock_container): + try: + context.snapshot_result = _build_snapshot(plan_id) + context.snapshot_error = None + except Exception as exc: + context.snapshot_result = None + context.snapshot_error = exc + + +@when("_build_snapshot is called") +def step_when_build_snapshot(context: Context) -> None: + mock_container = MagicMock() + mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle + + with patch("wf04_snapshot_helper.get_container", return_value=mock_container): + try: + context.snapshot_result = _build_snapshot(context.mock_plan_id) + context.snapshot_error = None + except Exception as exc: + context.snapshot_result = None + context.snapshot_error = exc + + +@then('the snapshot error should be ValueError containing "{fragment}"') +def step_then_valueerror_raised(context: Context, fragment: str) -> None: + assert context.snapshot_error is not None, ( + "Expected an exception but none was raised" + ) + assert isinstance(context.snapshot_error, ValueError), ( + f"Expected ValueError, got {type(context.snapshot_error).__name__}" + ) + assert fragment in str(context.snapshot_error), ( + f"Expected {fragment!r} in error message: {context.snapshot_error}" + ) + + +@then("the snapshot should have subplan_count {expected:d}") +def step_then_snapshot_subplan_count(context: Context, expected: int) -> None: + assert context.snapshot_result is not None, "Snapshot should not be None" + assert context.snapshot_result["subplan_count"] == expected, ( + f"Expected subplan_count={expected}, got {context.snapshot_result['subplan_count']}" + ) + + +@then("the snapshot subplans list should be empty") +def step_then_snapshot_subplans_empty(context: Context) -> None: + assert context.snapshot_result is not None + assert context.snapshot_result["subplans"] == [], ( + f"Expected empty subplans, got {context.snapshot_result['subplans']}" + ) + + +@then("each subplan should have mapped_projects populated") +def step_then_subplan_mapped_projects(context: Context) -> None: + assert context.snapshot_result is not None + for sp in context.snapshot_result["subplans"]: + assert "mapped_projects" in sp, "Subplan missing mapped_projects field" + assert len(sp["mapped_projects"]) > 0, ( + f"Subplan {sp['subplan_id']} has empty mapped_projects" + ) + + +@then("each subplan should have an unmapped_resources field") +def step_then_subplan_unmapped_field(context: Context) -> None: + assert context.snapshot_result is not None + for sp in context.snapshot_result["subplans"]: + assert "unmapped_resources" in sp, "Subplan missing unmapped_resources field" + + +@then("the subplan unmapped_resources should contain the unknown resource ID") +def step_then_subplan_unmapped_contains(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) == 1 + assert "res-unknown" in subplans[0]["unmapped_resources"], ( + f"Expected 'res-unknown' in unmapped_resources: {subplans[0]['unmapped_resources']}" + ) + + +# --------------------------------------------------------------------------- +# Additional _build_snapshot() assertions +# --------------------------------------------------------------------------- + + +@then("the snapshot plan_id should match the mocked plan ID") +def step_then_snapshot_plan_id_matches(context: Context) -> None: + assert context.snapshot_result is not None + assert context.snapshot_result["plan_id"] == context.mock_plan_id, ( + f"Expected plan_id={context.mock_plan_id!r}, " + f"got {context.snapshot_result['plan_id']!r}" + ) + + +@then("the snapshot project_scopes should be an empty list") +def step_then_snapshot_project_scopes_empty(context: Context) -> None: + assert context.snapshot_result is not None + assert context.snapshot_result["project_scopes"] == [], ( + f"Expected empty project_scopes, got {context.snapshot_result['project_scopes']}" + ) + + +@then("the snapshot validation_summary should be None") +def step_then_snapshot_validation_summary_none(context: Context) -> None: + assert context.snapshot_result is not None + assert context.snapshot_result["validation_summary"] is None, ( + f"Expected None validation_summary, " + f"got {context.snapshot_result['validation_summary']!r}" + ) + + +@then('subplan "{subplan_id}" should map to project "{project_name}"') +def step_then_subplan_maps_to_project( + context: Context, subplan_id: str, project_name: str +) -> None: + assert context.snapshot_result is not None + matching = [ + sp + for sp in context.snapshot_result["subplans"] + if sp["subplan_id"] == subplan_id + ] + assert len(matching) == 1, ( + f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}" + ) + assert project_name in matching[0]["mapped_projects"], ( + f"Expected {project_name!r} in mapped_projects of {subplan_id}, " + f"got {matching[0]['mapped_projects']}" + ) + + +@then('the iso result should be exactly "{expected}"') +def step_then_iso_result_exact(context: Context, expected: str) -> None: + assert context.iso_result == expected, ( + f"Expected exactly {expected!r}, got {context.iso_result!r}" + ) + + +@then('subplan "{subplan_id}" status should be "{expected}"') +def step_then_subplan_status(context: Context, subplan_id: str, expected: str) -> None: + assert context.snapshot_result is not None + matching = [ + sp + for sp in context.snapshot_result["subplans"] + if sp["subplan_id"] == subplan_id + ] + assert len(matching) == 1, ( + f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}" + ) + assert matching[0]["status"] == expected, ( + f"Expected status={expected!r} for {subplan_id}, got {matching[0]['status']!r}" + ) + + +@then('subplan "{subplan_id}" child_phase should be "{expected}"') +def step_then_subplan_child_phase( + context: Context, subplan_id: str, expected: str +) -> None: + assert context.snapshot_result is not None + matching = [ + sp + for sp in context.snapshot_result["subplans"] + if sp["subplan_id"] == subplan_id + ] + assert len(matching) == 1, ( + f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}" + ) + assert matching[0]["child_phase"] == expected, ( + f"Expected child_phase={expected!r} for {subplan_id}, " + f"got {matching[0]['child_phase']!r}" + ) + + +@then('subplan "{subplan_id}" started_at should be non-empty') +def step_then_subplan_started_at_non_empty(context: Context, subplan_id: str) -> None: + assert context.snapshot_result is not None + matching = [ + sp + for sp in context.snapshot_result["subplans"] + if sp["subplan_id"] == subplan_id + ] + assert len(matching) == 1, ( + f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}" + ) + assert matching[0]["started_at"] != "", ( + f"Expected non-empty started_at for {subplan_id}, " + f"got {matching[0]['started_at']!r}" + ) + + +@then( + 'subplan "{subplan_id}" child_validation_summary required_passed should be {expected:d}' +) +def step_then_subplan_validation_required_passed( + context: Context, subplan_id: str, expected: int +) -> None: + assert context.snapshot_result is not None + matching = [ + sp + for sp in context.snapshot_result["subplans"] + if sp["subplan_id"] == subplan_id + ] + assert len(matching) == 1, ( + f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}" + ) + summary = matching[0].get("child_validation_summary") + assert summary is not None, ( + f"Expected child_validation_summary for {subplan_id}, got None" + ) + actual = int(summary.get("required_passed", 0)) + assert actual == expected, ( + f"Expected required_passed={expected} for {subplan_id}, got {actual}" + ) + + +# --------------------------------------------------------------------------- +# _build_snapshot() child plan is None scenario +# --------------------------------------------------------------------------- + + +@given("a mocked lifecycle service with a subplan whose child plan is None") +def step_given_mock_plan_child_none(context: Context) -> None: + scope1 = _make_mock_project_scope("proj-a", ["res-1"]) + metadata = MagicMock() + metadata.project_scopes = [scope1] + + status1 = _make_mock_subplan_status("SUB_NONE", target_resources=["res-1"]) + plan = _make_mock_plan( + subplan_statuses=[status1], + multi_project_metadata=metadata, + ) + + lifecycle = MagicMock() + # Parent plan found, but get_plan returns None for the child subplan + lifecycle.get_plan.side_effect = lambda pid: ( + plan if pid == plan.identity.plan_id else None + ) + + context.mock_lifecycle = lifecycle + context.mock_plan_id = plan.identity.plan_id + + +@then("the subplan child_phase should be an empty string") +def step_then_child_phase_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["child_phase"] == "", ( + f"Expected empty child_phase, got {subplans[0]['child_phase']!r}" + ) + + +@then("the subplan child_state should be an empty string") +def step_then_child_state_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["child_state"] == "", ( + f"Expected empty child_state, got {subplans[0]['child_state']!r}" + ) + + +@then("the subplan child_updated_at should be an empty string") +def step_then_child_updated_at_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["child_updated_at"] == "", ( + f"Expected empty child_updated_at, got {subplans[0]['child_updated_at']!r}" + ) + + +@then("the subplan execute_started_at should be an empty string") +def step_then_execute_started_at_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["execute_started_at"] == "", ( + f"Expected empty execute_started_at, got {subplans[0]['execute_started_at']!r}" + ) + + +@then("the subplan execute_completed_at should be an empty string") +def step_then_execute_completed_at_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["execute_completed_at"] == "", ( + f"Expected empty execute_completed_at, got {subplans[0]['execute_completed_at']!r}" + ) + + +@then("the subplan apply_started_at should be an empty string") +def step_then_apply_started_at_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["apply_started_at"] == "", ( + f"Expected empty apply_started_at, got {subplans[0]['apply_started_at']!r}" + ) + + +@then("the subplan applied_at should be an empty string") +def step_then_applied_at_empty(context: Context) -> None: + assert context.snapshot_result is not None + subplans = context.snapshot_result["subplans"] + assert len(subplans) >= 1 + assert subplans[0]["applied_at"] == "", ( + f"Expected empty applied_at, got {subplans[0]['applied_at']!r}" + ) diff --git a/features/wf04_snapshot_helper.feature b/features/wf04_snapshot_helper.feature new file mode 100644 index 000000000..709ba3a3e --- /dev/null +++ b/features/wf04_snapshot_helper.feature @@ -0,0 +1,141 @@ +@phase1 @e2e_helpers @wf04 +Feature: WF04 Snapshot Helper Utilities + As a developer maintaining the WF04 E2E test infrastructure + I want unit tests for the snapshot helper's utility functions + So that timestamp normalisation, enum serialisation, decision counting, + and snapshot building are verified without running a full E2E test + + # --- _iso() timestamp normalisation --- + + @iso_timestamp + Scenario: _iso returns empty string for None + Given a None timestamp value + When _iso is called + Then the iso result should be an empty string + + @iso_timestamp + Scenario: _iso returns empty string for non-datetime truthy value + Given a non-datetime truthy value "not-a-date" + When _iso is called + Then the iso result should be an empty string + + @iso_timestamp + Scenario: _iso normalises naive datetime to UTC + Given a naive datetime 2026-03-15T10:30:00 + When _iso is called + Then the iso result should be exactly "2026-03-15T10:30:00+00:00" + + @iso_timestamp + Scenario: _iso normalises aware datetime to UTC + Given an aware datetime 2026-03-15T10:30:00 in timezone UTC+05:00 + When _iso is called + Then the iso result should be exactly "2026-03-15T05:30:00+00:00" + + @iso_timestamp + Scenario: _iso produces consistent format for mixed timezone inputs + Given a naive datetime 2026-03-15T12:00:00 + And an aware datetime 2026-03-15T12:00:00 in timezone UTC + When _iso is called on both + Then both iso results should be identical + + # --- _enum_value() serialisation --- + + @enum_value + Scenario: _enum_value returns .value for enum-like objects + Given an enum-like object with value "completed" + When _enum_value is called + Then the enum result should be "completed" + + @enum_value + Scenario: _enum_value returns str for plain objects + Given a plain string "hello" + When _enum_value is called + Then the enum result should be "hello" + + # --- count_decision_nodes() --- + + @decision_count + Scenario: count_decision_nodes returns 0 for empty input + Given an empty list tree + When count_decision_nodes is called + Then the decision node count should be 0 + + @decision_count + Scenario: count_decision_nodes counts single root node + Given a tree with one decision node + When count_decision_nodes is called + Then the decision node count should be 1 + + @decision_count + Scenario: count_decision_nodes counts nested children + Given a tree with a root and 2 child decision nodes + When count_decision_nodes is called + Then the decision node count should be 3 + + @decision_count + Scenario: count_decision_nodes skips nodes without decision_id + Given a tree with mixed decision and non-decision nodes + When count_decision_nodes is called + Then the decision node count should be 2 + + @decision_count + Scenario: count_decision_nodes handles list input + Given a list of 2 independent decision trees + When count_decision_nodes is called + Then the decision node count should be 2 + + @decision_count + Scenario: count_decision_nodes truncates at max_depth + Given a tree with depth 5 and one decision node per level + When count_decision_nodes is called with max_depth 3 + Then the decision node count should be 3 + + # --- _build_snapshot() with mocked lifecycle service --- + + @snapshot_build + Scenario: _build_snapshot raises ValueError for missing plan + Given a mocked lifecycle service that returns None for plan lookup + When _build_snapshot is called with plan_id "nonexistent" + Then the snapshot error should be ValueError containing "not found" + + @snapshot_build + Scenario: _build_snapshot returns correct structure for plan without subplans + Given a mocked lifecycle service with a plan that has no subplans + When _build_snapshot is called + Then the snapshot should have subplan_count 0 + And the snapshot subplans list should be empty + And the snapshot plan_id should match the mocked plan ID + And the snapshot project_scopes should be an empty list + And the snapshot validation_summary should be None + + @snapshot_build + Scenario: _build_snapshot maps resources to projects for subplans + Given a mocked lifecycle service with a plan that has 2 subplans and project scopes + When _build_snapshot is called + Then the snapshot should have subplan_count 2 + And subplan "SUB01" should map to project "proj-a" + And subplan "SUB02" should map to project "proj-b" + And each subplan should have mapped_projects populated + And each subplan should have an unmapped_resources field + And subplan "SUB01" status should be "completed" + And subplan "SUB01" child_phase should be "completed" + And subplan "SUB01" started_at should be non-empty + And subplan "SUB01" child_validation_summary required_passed should be 1 + + @snapshot_build + Scenario: _build_snapshot includes unmapped_resources for unknown resource IDs + Given a mocked lifecycle service with a subplan targeting an unmapped resource + When _build_snapshot is called + Then the subplan unmapped_resources should contain the unknown resource ID + + @snapshot_build + Scenario: _build_snapshot defaults child fields when child plan is None + Given a mocked lifecycle service with a subplan whose child plan is None + When _build_snapshot is called + Then the subplan child_phase should be an empty string + And the subplan child_state should be an empty string + And the subplan child_updated_at should be an empty string + And the subplan execute_started_at should be an empty string + And the subplan execute_completed_at should be an empty string + And the subplan apply_started_at should be an empty string + And the subplan applied_at should be an empty string diff --git a/robot/e2e/wf04_multi_project.robot b/robot/e2e/wf04_multi_project.robot new file mode 100644 index 000000000..c4bdc9ed7 --- /dev/null +++ b/robot/e2e/wf04_multi_project.robot @@ -0,0 +1,577 @@ +*** Settings *** +Documentation E2E test for Workflow Example 4: Multi-Project Dependency Update. +... +... Advanced scenario using the supervised automation profile. +... Three microservices share a common library with a breaking +... change (v1 to v2). CleverAgents creates a parent plan +... targeting all 4 projects, spawns child subplans per project, +... executes in dependency order (common-lib first, then services), +... validates per project, and applies in dependency order. +... +... Zero mocking — real CLI, real LLM API keys. +Resource common_e2e.resource +Suite Setup WF04 Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Variables *** +${ACTION_BASE} local/wf04-dep-update +${LIB_BASE} wf04-common-lib +${SVC1_BASE} wf04-svc-auth +${SVC2_BASE} wf04-svc-billing +${SVC3_BASE} wf04-svc-gateway +${WF04_SNAPSHOT_HELPER} ${CURDIR}${/}wf04_snapshot_helper.py + +*** Keywords *** +WF04 Suite Setup + [Documentation] E2E Suite Setup plus unique suffix generation and actor selection. + E2E Suite Setup + # Initialise the database so commands work in all tests. + ${init}= Run CleverAgents Command init --force --yes + Should Be Equal As Integers ${init.rc} 0 + Should Not Contain ${init.stdout}${init.stderr} Traceback + Should Not Contain ${init.stdout}${init.stderr} INTERNAL + # Generate a unique suffix for resource/project names to avoid + # UNIQUE constraint collisions on repeated or parallel CI runs. + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + Set Suite Variable ${RUN_SUFFIX} ${suffix} + # Derive unique names from base + suffix + Set Suite Variable ${ACTION_NAME} ${ACTION_BASE}-${suffix} + Set Suite Variable ${LIB_RESOURCE} ${LIB_BASE}-res-${suffix} + Set Suite Variable ${SVC1_RESOURCE} ${SVC1_BASE}-res-${suffix} + Set Suite Variable ${SVC2_RESOURCE} ${SVC2_BASE}-res-${suffix} + Set Suite Variable ${SVC3_RESOURCE} ${SVC3_BASE}-res-${suffix} + Set Suite Variable ${LIB_PROJECT} ${LIB_BASE}-proj-${suffix} + Set Suite Variable ${SVC1_PROJECT} ${SVC1_BASE}-proj-${suffix} + Set Suite Variable ${SVC2_PROJECT} ${SVC2_BASE}-proj-${suffix} + Set Suite Variable ${SVC3_PROJECT} ${SVC3_BASE}-proj-${suffix} + # Pick an actor that matches the available API key. + ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) + ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) + IF ${has_openai} + ${actor}= Set Variable openai/gpt-4o + ELSE IF ${has_anthropic} + ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 + ELSE + ${actor}= Set Variable openai/gpt-4o + END + Set Suite Variable ${LLM_ACTOR} ${actor} + +Create Library Repo + [Documentation] Create temp git repo for the common library. + ${repo}= Create Temp Git Repo ${LIB_BASE}-${RUN_SUFFIX} + Create Directory ${repo}${/}src + ${lib_content}= Catenate SEPARATOR=\n + ... """Common library v1 — shared utilities.""" + ... ${EMPTY} + ... ${EMPTY} + ... __version__ = "1.0.0" + ... ${EMPTY} + ... ${EMPTY} + ... def connect(host, port): + ... ${SPACE}${SPACE}${SPACE}${SPACE}"""Connect to a service (v1 API).""" + ... ${SPACE}${SPACE}${SPACE}${SPACE}return {"host": host, "port": port, "status": "connected"} + Create File ${repo}${/}src${/}client.py ${lib_content} + ${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr} + ${git_commit}= Run Process git commit -m Initial common library v1 cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr} + RETURN ${repo} + +Create Service Repo + [Documentation] Create temp git repo for a microservice. + [Arguments] ${name} ${import_line} + ${repo}= Create Temp Git Repo ${name}-${RUN_SUFFIX} + Create Directory ${repo}${/}src + ${svc_content}= Catenate SEPARATOR=\n + ... """${name} service — depends on common-lib v1.""" + ... ${import_line} + ... ${EMPTY} + ... ${EMPTY} + ... def start(): + ... ${SPACE}${SPACE}${SPACE}${SPACE}conn = connect("localhost", 8080) + ... ${SPACE}${SPACE}${SPACE}${SPACE}return conn + Create File ${repo}${/}src${/}app.py ${svc_content} + Create File ${repo}${/}requirements.txt common-lib==1.0.0\n + ${git_add}= Run Process git add . cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 git add failed: ${git_add.stderr} + ${git_commit}= Run Process git commit -m Initial ${name} service cwd=${repo} timeout=30s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 git commit failed: ${git_commit.stderr} + RETURN ${repo} + +Register Resource And Project + [Documentation] Register a git-checkout resource and create a project. + [Arguments] ${resource_name} ${project_name} ${repo_dir} + ${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo_dir} timeout=30s on_timeout=kill + Should Be Equal As Integers ${branch_result.rc} 0 git rev-parse failed: ${branch_result.stderr} + ${branch}= Strip String ${branch_result.stdout} + ${r_res}= Run CleverAgents Command + ... resource add git-checkout ${resource_name} + ... --path ${repo_dir} --branch ${branch} + Should Be Equal As Integers ${r_res.rc} 0 resource add failed: ${r_res.stderr} + Should Not Contain ${r_res.stdout}${r_res.stderr} Traceback + Should Not Contain ${r_res.stdout}${r_res.stderr} INTERNAL + ${r_proj}= Run CleverAgents Command + ... project create ${project_name} + ... --resource ${resource_name} + Should Be Equal As Integers ${r_proj.rc} 0 project create failed: ${r_proj.stderr} + Should Not Contain ${r_proj.stdout}${r_proj.stderr} Traceback + Should Not Contain ${r_proj.stdout}${r_proj.stderr} INTERNAL + +WF04 Test Teardown + [Documentation] Log diagnostic context on failure for debugging. + ... Captures plan status and decision tree so CI failures + ... in this 25-minute LLM-dependent test have actionable data. + ${plan_id}= Get Variable Value ${WF04_PLAN_ID} ${EMPTY} + IF '${plan_id}' != '' + ${status} ${result}= Run Keyword And Ignore Error + ... Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=30s + IF '${status}' == 'PASS' + Log Teardown plan status: ${result.stdout} WARN + END + ${tree_status} ${tree_result}= Run Keyword And Ignore Error + ... Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=30s + IF '${tree_status}' == 'PASS' + Log Teardown plan tree: ${tree_result.stdout} WARN + END + END + +Register Validation For Project + [Documentation] Create a validation YAML, register it, and attach it to a project's resource. + [Arguments] ${validation_name} ${resource_name} ${project_name} + ${val_yaml}= Catenate SEPARATOR=\n + ... name: ${validation_name} + ... description: "Simple pass-through validation for E2E testing" + ... source: custom + ... mode: required + ... code: | + ... ${SPACE}${SPACE}return {"passed": True, "data": {}, "message": "Validation passed"} + ... input_schema: + ... ${SPACE}${SPACE}type: object + ... ${SPACE}${SPACE}properties: {} + ... timeout: 30 + ${val_path}= Set Variable ${SUITE_HOME}${/}${validation_name}.yaml + Create File ${val_path} ${val_yaml} + ${r_add}= Run CleverAgents Command + ... validation add --config ${val_path} expected_rc=None + Log Validation add rc=${r_add.rc} stdout=${r_add.stdout} stderr=${r_add.stderr} + Should Be Equal As Integers ${r_add.rc} 0 + ... validation add failed (rc=${r_add.rc}): ${r_add.stderr} + Should Not Contain ${r_add.stdout}${r_add.stderr} Traceback + Should Not Contain ${r_add.stdout}${r_add.stderr} INTERNAL + # Positional args: (resource first, validation second) + ${r_attach}= Run CleverAgents Command + ... validation attach --project ${project_name} + ... ${resource_name} ${validation_name} expected_rc=None + Log Validation attach rc=${r_attach.rc} stdout=${r_attach.stdout} stderr=${r_attach.stderr} + Should Be Equal As Integers ${r_attach.rc} 0 + ... validation attach failed for ${project_name} (rc=${r_attach.rc}): ${r_attach.stderr} + Should Not Contain ${r_attach.stdout}${r_attach.stderr} Traceback + Should Not Contain ${r_attach.stdout}${r_attach.stderr} INTERNAL + +Attach Validation To Project + [Documentation] Attach an already-registered validation to a project's resource. + [Arguments] ${validation_name} ${resource_name} ${project_name} + # Positional args: (resource first, validation second) + ${r_attach}= Run CleverAgents Command + ... validation attach --project ${project_name} + ... ${resource_name} ${validation_name} expected_rc=None + Log Validation attach rc=${r_attach.rc} stdout=${r_attach.stdout} stderr=${r_attach.stderr} + Should Be Equal As Integers ${r_attach.rc} 0 + ... validation attach failed for ${project_name} (rc=${r_attach.rc}): ${r_attach.stderr} + Should Not Contain ${r_attach.stdout}${r_attach.stderr} Traceback + Should Not Contain ${r_attach.stdout}${r_attach.stderr} INTERNAL + +Parse Json Payload + [Documentation] Parse JSON object/array from stdout with optional log preamble. + ... Delegates to ``Extract JSON From Stdout`` which uses + ... ``json.JSONDecoder().raw_decode()`` for robustness against + ... trailing non-JSON output. + [Arguments] ${stdout} + ${parsed}= Extract JSON From Stdout ${stdout} + RETURN ${parsed} + +Get WF04 Plan Snapshot + [Documentation] Read parent/subplan metadata for deterministic WF04 assertions. + [Arguments] ${plan_id} + ${snapshot_result}= Run Process + ... ${PYTHON} ${WF04_SNAPSHOT_HELPER} ${plan_id} + ... cwd=${SUITE_HOME} timeout=120s on_timeout=kill + ... env:CLEVERAGENTS_HOME=${SUITE_HOME} + ... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true + ... env:NO_COLOR=1 + ... env:PYTHONPATH=${WORKSPACE}${/}src + Should Be Equal As Integers ${snapshot_result.rc} 0 + ... Snapshot helper failed (rc=${snapshot_result.rc}): ${snapshot_result.stderr} + Should Not Be Empty ${snapshot_result.stdout} + ${snapshot}= Parse Json Payload ${snapshot_result.stdout} + RETURN ${snapshot} + +Verify WF04 Child Plan Spawning + [Documentation] AC-4: verify exactly 4 child plans mapped one-per-project. + [Arguments] ${snapshot} + ${subplans}= Evaluate $snapshot.get('subplans', []) + ${subplan_count}= Evaluate len($subplans) + IF ${subplan_count} == 0 + Skip LLM produced no child plans — AC-4 child-plan mapping assertions were not exercised + END + Should Be Equal As Integers ${subplan_count} 4 + ... Expected exactly 4 child plans (common-lib + 3 services), found ${subplan_count} + ${mapped_projects}= Evaluate sorted({p for sp in $subplans for p in sp.get('mapped_projects', []) if p}) + ${expected_projects}= Create List ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT} + ${expected_sorted}= Evaluate sorted($expected_projects) + ${all_projects_covered}= Evaluate $mapped_projects == $expected_sorted + Should Be True ${all_projects_covered} + ... Child plans should cover all 4 projects. expected=${expected_sorted} actual=${mapped_projects} + ${single_mapping}= Evaluate all(len(sp.get('mapped_projects', [])) == 1 for sp in $subplans) + Should Be True ${single_mapping} + ... Each child plan should map to exactly one project scope + +Verify WF04 Execution Order + [Documentation] AC-5: common-lib executes before service subplans. + [Arguments] ${snapshot} + ${subplans}= Evaluate $snapshot.get('subplans', []) + ${subplan_count}= Evaluate len($subplans) + IF ${subplan_count} == 0 + Skip LLM produced no child plans — AC-5 execution-order assertions were not exercised + END + ${lib_subplans}= Evaluate [sp for sp in $subplans if '${LIB_PROJECT}' in sp.get('mapped_projects', [])] + ${lib_count}= Evaluate len($lib_subplans) + Should Be Equal As Integers ${lib_count} 1 + ... Expected exactly one common-lib child plan, found ${lib_count} + ${lib_completed}= Evaluate $lib_subplans[0].get('completed_at') or $lib_subplans[0].get('execute_completed_at') or '' + Should Not Be Empty ${lib_completed} + ... common-lib child plan completion timestamp is required for ordering checks + ${service_projects}= Create List ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT} + ${svc_subplans}= Evaluate [sp for sp in $subplans if any(p in $service_projects for p in sp.get('mapped_projects', []))] + ${svc_count}= Evaluate len($svc_subplans) + Should Be Equal As Integers ${svc_count} 3 + ... Expected exactly three service child plans, found ${svc_count} + ${svc_starts_present}= Evaluate all((sp.get('started_at') or sp.get('execute_started_at') or '') != '' for sp in $svc_subplans) + Should Be True ${svc_starts_present} + ... Service child plans must expose start timestamps for execution-order checks + ${services_after_lib}= Evaluate all((sp.get('started_at') or sp.get('execute_started_at')) >= $lib_completed for sp in $svc_subplans) + Should Be True ${services_after_lib} + ... Service execution must start only after common-lib execution completes + +Verify WF04 Validation Outcomes + [Documentation] AC-6: each child plan must report validation pass results. + [Arguments] ${snapshot} + ${subplans}= Evaluate $snapshot.get('subplans', []) + ${subplan_count}= Evaluate len($subplans) + IF ${subplan_count} == 0 + Skip LLM produced no child plans — AC-6 per-project validation assertions were not exercised + END + ${validation_present}= Evaluate all(isinstance(sp.get('child_validation_summary'), dict) and len(sp.get('child_validation_summary')) > 0 for sp in $subplans) + Should Be True ${validation_present} + ... Each child plan must expose a non-empty validation summary + ${validation_passed}= Evaluate all(int((sp.get('child_validation_summary') or {}).get('required_passed', 0) or 0) >= 1 and int((sp.get('child_validation_summary') or {}).get('required_failed', 0) or 0) == 0 for sp in $subplans) + Should Be True ${validation_passed} + ... Each child plan must pass required validations (required_failed == 0) + +Verify WF04 Apply Order + [Documentation] AC-7: common-lib apply must complete before service applies. + [Arguments] ${snapshot} + ${subplans}= Evaluate $snapshot.get('subplans', []) + ${subplan_count}= Evaluate len($subplans) + IF ${subplan_count} == 0 + Skip LLM produced no child plans — AC-7 apply-order assertions were not exercised + END + ${lib_subplans}= Evaluate [sp for sp in $subplans if '${LIB_PROJECT}' in sp.get('mapped_projects', [])] + ${lib_count}= Evaluate len($lib_subplans) + Should Be Equal As Integers ${lib_count} 1 + ... Expected exactly one common-lib child plan, found ${lib_count} + ${lib_applied}= Evaluate $lib_subplans[0].get('applied_at') or $lib_subplans[0].get('child_updated_at') or '' + Should Not Be Empty ${lib_applied} + ... common-lib child plan apply timestamp is required for apply-order checks + ${service_projects}= Create List ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT} + ${svc_subplans}= Evaluate [sp for sp in $subplans if any(p in $service_projects for p in sp.get('mapped_projects', []))] + ${svc_count}= Evaluate len($svc_subplans) + Should Be Equal As Integers ${svc_count} 3 + ... Expected exactly three service child plans, found ${svc_count} + ${svc_apply_present}= Evaluate all((sp.get('applied_at') or sp.get('child_updated_at') or '') != '' for sp in $svc_subplans) + Should Be True ${svc_apply_present} + ... Service child plans must expose apply/updated timestamps for apply-order checks + ${services_after_lib_apply}= Evaluate all((sp.get('applied_at') or sp.get('child_updated_at')) >= $lib_applied for sp in $svc_subplans) + Should Be True ${services_after_lib_apply} + ... Service applies must occur after common-lib apply + +Count Decision Nodes + [Documentation] Recursively count decision nodes in a plan tree JSON structure. + ... Invokes ``wf04_snapshot_helper.py --count-nodes`` as a subprocess + ... to avoid importing the application DI container into the Robot + ... test runner process. + [Arguments] ${tree_payload} + ${tree_json}= Evaluate __import__('json').dumps($tree_payload) + ${tmp_path}= Evaluate __import__('tempfile').NamedTemporaryFile(mode='w', suffix='.json', delete=False).name + Evaluate __import__('pathlib').Path(r'${tmp_path}').write_text($tree_json, encoding='utf-8') + ${result}= Run Process + ... ${PYTHON} ${WF04_SNAPSHOT_HELPER} --count-nodes ${tmp_path} + ... timeout=30s on_timeout=kill + Evaluate __import__('os').unlink(r'${tmp_path}') + Should Be Equal As Integers ${result.rc} 0 + ... count-nodes failed (rc=${result.rc}): ${result.stderr} + ${count}= Convert To Integer ${result.stdout.strip()} + RETURN ${count} + +Verify Plan In Lifecycle List + [Documentation] Verify a plan appears in lifecycle-list output. + [Arguments] ${plan_id} + ${list_result}= Run CleverAgents Command + ... plan lifecycle-list --format json expected_rc=None timeout=120s + Should Be Equal As Integers ${list_result.rc} 0 + ... lifecycle-list failed (rc=${list_result.rc}): ${list_result.stderr} + Output Should Contain ${list_result} ${plan_id} + +*** Test Cases *** +WF04 Multi Project Dependency Update Supervised Profile + [Documentation] Full supervised-profile workflow: register 4 repos, + ... create multi-project action with invariants, plan use + ... targeting all 4 projects with --automation-profile supervised, + ... execute with child plan spawning in dependency order, + ... verify per-project validation, and apply in dependency order. + [Timeout] 25 minutes + [Teardown] WF04 Test Teardown + Skip If No LLM Keys + # Initialise test variable for teardown access. + Set Test Variable ${WF04_PLAN_ID} ${EMPTY} + + # ---- Create fixture repos ---- + ${lib_repo}= Create Library Repo + ${svc1_repo}= Create Service Repo svc-auth from common_lib.client import connect + ${svc2_repo}= Create Service Repo svc-billing from common_lib.client import connect + ${svc3_repo}= Create Service Repo svc-gateway from common_lib.client import connect + + # ---- Register resources and projects ---- + Register Resource And Project ${LIB_RESOURCE} ${LIB_PROJECT} ${lib_repo} + Register Resource And Project ${SVC1_RESOURCE} ${SVC1_PROJECT} ${svc1_repo} + Register Resource And Project ${SVC2_RESOURCE} ${SVC2_PROJECT} ${svc2_repo} + Register Resource And Project ${SVC3_RESOURCE} ${SVC3_PROJECT} ${svc3_repo} + + # ---- Register and attach validations for all 4 projects (AC-6) ---- + ${val_name}= Set Variable local/wf04-val-${RUN_SUFFIX} + Register Validation For Project ${val_name} ${LIB_RESOURCE} ${LIB_PROJECT} + # Validation already registered; attach to remaining 3 projects + Attach Validation To Project ${val_name} ${SVC1_RESOURCE} ${SVC1_PROJECT} + Attach Validation To Project ${val_name} ${SVC2_RESOURCE} ${SVC2_PROJECT} + Attach Validation To Project ${val_name} ${SVC3_RESOURCE} ${SVC3_PROJECT} + + # ---- Create action with supervised profile and invariants ---- + ${action_yaml}= Catenate SEPARATOR=\n + ... name: ${ACTION_NAME} + ... description: Update common-lib from v1 to v2 across all dependent services + ... definition_of_done: All services updated to use common-lib v2 API + ... strategy_actor: ${LLM_ACTOR} + ... execution_actor: ${LLM_ACTOR} + ... automation_profile: supervised + ... reusable: true + ... state: available + ... invariants: + ... ${SPACE}${SPACE}- "Each dependent project must be updated in its own child plan" + ... ${SPACE}${SPACE}- "All child plans must pass validation before any can be applied" + ... ${SPACE}${SPACE}- "The library update in common-lib must be applied first" + ${action_path}= Set Variable ${SUITE_HOME}${/}wf04_action.yaml + Create File ${action_path} ${action_yaml} + ${r_action}= Run CleverAgents Command + ... action create --config ${action_path} + Should Be Equal As Integers ${r_action.rc} 0 + ... action create failed (rc=${r_action.rc}): ${r_action.stderr} + Should Not Contain ${r_action.stdout}${r_action.stderr} Traceback + Should Not Contain ${r_action.stdout}${r_action.stderr} INTERNAL + Output Should Contain ${r_action} ${ACTION_NAME} + + # ---- Plan use targeting ALL 4 projects with supervised profile (AC-3) ---- + ${r_use}= Run CleverAgents Command + ... plan use ${ACTION_NAME} + ... ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT} + ... --automation-profile supervised + ... --format json + ... timeout=120s + Should Be Equal As Integers ${r_use.rc} 0 + ... plan use failed (rc=${r_use.rc}): ${r_use.stderr} + Should Not Contain ${r_use.stdout}${r_use.stderr} Traceback + Should Not Contain ${r_use.stdout}${r_use.stderr} INTERNAL + ${use_payload}= Parse Json Payload ${r_use.stdout} + ${plan_id}= Evaluate str($use_payload.get('plan_id', '')) + Should Not Be Empty ${plan_id} msg=Expected plan_id in plan use JSON output + Log Plan ID: ${plan_id} + Set Test Variable ${WF04_PLAN_ID} ${plan_id} + + # Verify plan targets all 4 projects exactly (AC-3) + ${use_projects}= Evaluate sorted([link.get('project_name', '') for link in $use_payload.get('project_links', []) if isinstance(link, dict)]) + ${expected_projects}= Create List ${LIB_PROJECT} ${SVC1_PROJECT} ${SVC2_PROJECT} ${SVC3_PROJECT} + ${expected_sorted}= Evaluate sorted($expected_projects) + ${projects_match}= Evaluate $use_projects == $expected_sorted + Should Be True ${projects_match} + ... plan use should target all 4 projects. expected=${expected_sorted} actual=${use_projects} + + # ---- Strategize ---- + # Supervised profile requires two explicit ``plan execute`` calls: + # the first advances the plan through strategize, the second runs + # actual execution. Both use the same CLI command. + ${r_strat}= Run CleverAgents Command + ... plan execute ${plan_id} + ... --format json expected_rc=None timeout=180s + Log Strategize rc=${r_strat.rc} stdout=${r_strat.stdout} stderr=${r_strat.stderr} + Should Not Contain ${r_strat.stdout}${r_strat.stderr} Traceback + Should Not Contain ${r_strat.stdout}${r_strat.stderr} INTERNAL + IF ${r_strat.rc} != 0 + Fail plan execute (strategize) failed (rc=${r_strat.rc}): ${r_strat.stderr} + END + + # ---- Decision tree — verify child plan spawning (AC-4) ---- + ${r_tree}= Run CleverAgents Command + ... plan tree ${plan_id} --format json + ... expected_rc=None timeout=60s + Should Be Equal As Integers ${r_tree.rc} 0 + ... plan tree failed (rc=${r_tree.rc}): ${r_tree.stderr} + Should Not Contain ${r_tree.stdout}${r_tree.stderr} Traceback + Should Not Contain ${r_tree.stdout}${r_tree.stderr} INTERNAL + Should Not Be Empty ${r_tree.stdout} Plan tree output should not be empty + Log Decision tree: ${r_tree.stdout} + + # Parse tree for child plan / decision structure + ${tree_payload}= Parse Json Payload ${r_tree.stdout} + ${decision_count}= Count Decision Nodes ${tree_payload} + Log Decision tree contains ${decision_count} decision node(s) + Should Be True ${decision_count} >= 1 + ... Plan tree should contain at least one decision node after strategize (found ${decision_count}) + # Some providers/runs expose a minimal strategize tree before execute. + # Child spawning is asserted deterministically after execute via snapshot. + + # ---- Execute — dependency-ordered execution (AC-5) ---- + ${r_exec}= Run CleverAgents Command + ... plan execute ${plan_id} + ... --format json expected_rc=None timeout=300s + Log Execute rc=${r_exec.rc} stdout=${r_exec.stdout} stderr=${r_exec.stderr} + Should Not Contain ${r_exec.stdout}${r_exec.stderr} Traceback + Should Not Contain ${r_exec.stdout}${r_exec.stderr} INTERNAL + IF ${r_exec.rc} != 0 + Fail plan execute failed (rc=${r_exec.rc}): ${r_exec.stderr} + END + + # Deterministic WF04 assertions after execute (AC-4/AC-5/AC-6) + ${exec_snapshot}= Get WF04 Plan Snapshot ${plan_id} + # Guard: if the snapshot contains zero subplans, skip the entire test rather + # than letting individual verification keywords silently skip all ACs. + # This ensures CI reports show SKIPPED (visible) rather than PASSED (misleading). + ${exec_subplan_count}= Evaluate int($exec_snapshot.get('subplan_count', 0)) + IF ${exec_subplan_count} == 0 + Skip LLM produced 0 subplans — AC-4/5/6/7 verification cannot be exercised (entire test skipped) + END + Verify WF04 Child Plan Spawning ${exec_snapshot} + Verify WF04 Execution Order ${exec_snapshot} + Verify WF04 Validation Outcomes ${exec_snapshot} + + # ---- Post-execute decision tree — verify child plan count (AC-4) ---- + ${r_tree_post}= Run CleverAgents Command + ... plan tree ${plan_id} --format json + ... expected_rc=None timeout=60s + Should Be Equal As Integers ${r_tree_post.rc} 0 + ... plan tree (post-execute) failed (rc=${r_tree_post.rc}): ${r_tree_post.stderr} + Should Not Contain ${r_tree_post.stdout}${r_tree_post.stderr} Traceback + Should Not Contain ${r_tree_post.stdout}${r_tree_post.stderr} INTERNAL + ${tree_post_payload}= Parse Json Payload ${r_tree_post.stdout} + ${post_exec_decision_count}= Count Decision Nodes ${tree_post_payload} + Log Post-execute decision tree contains ${post_exec_decision_count} decision node(s) + # Require non-trivial tree depth after execute. Subplan spawning is + # asserted deterministically via internal snapshot checks above. + Should Be True ${post_exec_decision_count} >= 2 + ... Plan tree should contain at least 2 decision nodes after execute (found ${post_exec_decision_count}) + # Execute should preserve or grow the tree — never shrink it. + Should Be True ${post_exec_decision_count} >= ${decision_count} + ... Post-execute decision count (${post_exec_decision_count}) should not be less than post-strategize count (${decision_count}) + + # ---- AC-6 verified above via per-child validation summaries ---- + + # ---- Plan lifecycle-list — verify plan exists (replaces deprecated plan list) ---- + Verify Plan In Lifecycle List ${plan_id} + + # ---- Verify plan status shows multi-project state ---- + ${r_status_mid}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=60s + Should Be Equal As Integers ${r_status_mid.rc} 0 + ... plan status failed (rc=${r_status_mid.rc}): ${r_status_mid.stderr} + Should Not Contain ${r_status_mid.stdout}${r_status_mid.stderr} Traceback + Should Not Contain ${r_status_mid.stdout}${r_status_mid.stderr} INTERNAL + Output Should Contain ${r_status_mid} ${plan_id} + # Check for automation profile reference in status + ${status_combined}= Set Variable ${r_status_mid.stdout}${r_status_mid.stderr} + ${status_lower}= Evaluate ($status_combined).lower() + ${has_profile_ref}= Evaluate 'supervised' in $status_lower or 'automation' in $status_lower or 'profile' in $status_lower + Should Be True ${has_profile_ref} + ... Plan status should reference automation profile (supervised) + # Parse automation_profile field for precise assertion + ${mid_profile}= Safe Parse Json Field ${r_status_mid.stdout} automation_profile + Should Not Be Empty ${mid_profile} + ... automation_profile field should be present in plan status JSON + Should Be Equal As Strings ${mid_profile} supervised + ... Plan automation profile should be supervised (found ${mid_profile}) + + # ---- Diff ---- + ${r_diff}= Run CleverAgents Command + ... plan diff ${plan_id} --format plain + ... expected_rc=None timeout=60s + Log Diff rc=${r_diff.rc} stdout=${r_diff.stdout} stderr=${r_diff.stderr} + Should Not Contain ${r_diff.stdout}${r_diff.stderr} Traceback + Should Not Contain ${r_diff.stdout}${r_diff.stderr} INTERNAL + Should Be Equal As Integers ${r_diff.rc} 0 + ... plan diff failed (rc=${r_diff.rc}): ${r_diff.stderr} + + # ---- Apply — dependency-ordered apply (AC-7) ---- + ${r_apply}= Run CleverAgents Command + ... plan lifecycle-apply ${plan_id} --yes --format json + ... expected_rc=None timeout=180s + Log Apply rc=${r_apply.rc} stdout=${r_apply.stdout} stderr=${r_apply.stderr} + Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback + Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL + IF ${r_apply.rc} == 0 + Output Should Contain ${r_apply} ${plan_id} + # Verify the plan transitioned — check for apply-phase indicators (AC-7) + ${apply_phase}= Safe Parse Json Field ${r_apply.stdout} phase + Should Not Be Empty ${apply_phase} + ... phase field should be present in lifecycle-apply JSON output + ${apply_phase_lower}= Evaluate ($apply_phase).lower() + Should Contain ${apply_phase_lower} apply + ... Plan phase should indicate apply after lifecycle-apply (found ${apply_phase}) + ${apply_snapshot}= Get WF04 Plan Snapshot ${plan_id} + # Guard: apply snapshot must also contain subplans for AC-7 verification. + # The exec guard above already skips the whole test if 0 subplans, so + # reaching this point implies subplans existed post-execute. If they + # disappeared post-apply, that would be a real regression worth flagging. + ${apply_subplan_count}= Evaluate int($apply_snapshot.get('subplan_count', 0)) + Should Be True ${apply_subplan_count} >= 1 + ... Snapshot reported 0 subplans after apply — subplans existed post-execute but vanished post-apply + Verify WF04 Apply Order ${apply_snapshot} + Verify WF04 Validation Outcomes ${apply_snapshot} + ELSE + Fail lifecycle-apply failed (rc=${r_apply.rc}) stdout=${r_apply.stdout} stderr=${r_apply.stderr} + END + + # ---- Verify final status ---- + ${r_status}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=60s + Should Be Equal As Integers ${r_status.rc} 0 + ... plan status (final) failed (rc=${r_status.rc}): ${r_status.stderr} + Should Not Contain ${r_status.stdout}${r_status.stderr} Traceback + Should Not Contain ${r_status.stdout}${r_status.stderr} INTERNAL + Should Not Be Empty ${r_status.stdout} Final plan status output should not be empty + Output Should Contain ${r_status} ${plan_id} + # Parse phase from final status and assert non-empty (proves lifecycle events processed) + ${final_phase}= Safe Parse Json Field ${r_status.stdout} phase + ${final_state}= Safe Parse Json Field ${r_status.stdout} processing_state + Log Final phase=${final_phase} processing_state=${final_state} + ${final_state_populated}= Evaluate $final_phase != '' or $final_state != '' + Should Be True ${final_state_populated} + ... Final plan status should have non-empty phase or processing_state + # After a full lifecycle (execute + apply), phase or state should reflect completion + ${final_lower}= Evaluate ($final_phase).lower() if $final_phase else '' + ${state_lower}= Evaluate ($final_state).lower() if $final_state else '' + ${is_terminal}= Evaluate 'apply' in $final_lower or 'complete' in $final_lower or 'done' in $final_lower or 'complete' in $state_lower or 'done' in $state_lower or 'applied' in $state_lower + Should Be True ${is_terminal} + ... Final status should indicate a terminal/applied state (phase=${final_phase}, state=${final_state}) + + # ---- Final: plan should still appear in lifecycle-list ---- + Verify Plan In Lifecycle List ${plan_id} diff --git a/robot/e2e/wf04_snapshot_helper.py b/robot/e2e/wf04_snapshot_helper.py new file mode 100644 index 000000000..49b62292d --- /dev/null +++ b/robot/e2e/wf04_snapshot_helper.py @@ -0,0 +1,225 @@ +"""Snapshot helper for WF04 E2E assertions. + +This script is executed by ``wf04_multi_project.robot`` to collect +deterministic parent/subplan metadata from the lifecycle service. +""" + +from __future__ import annotations + +import json +import sys +import traceback +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.append(str(SRC)) + +from cleveragents.application.container import get_container # noqa: E402 + + +def _iso(value: Any) -> str: + """Return ISO timestamp string normalised to UTC, or empty string. + + Ensures all timestamps are in a consistent UTC-aware format so that + lexicographic string comparison in Robot assertions is reliable. + """ + if value is None: + return "" + if not isinstance(value, datetime): + return "" + # Normalise naive datetimes by assuming UTC + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + # Convert to UTC and produce a consistent representation + return value.astimezone(UTC).isoformat() + + +def _enum_value(value: Any) -> str: + """Return enum ``.value`` when available, else ``str(value)``.""" + return value.value if hasattr(value, "value") else str(value) + + +DecisionTree = dict[str, Any] | list[Any] + + +def count_decision_nodes(root: DecisionTree | Any, *, max_depth: int = 50) -> int: + """Recursively count decision nodes in a plan tree structure. + + Walks the tree rooted at *root* and counts every ``dict`` node whose + ``decision_id`` field is non-empty. Works for both dict (single root) + and list (forest) inputs. + + A *max_depth* guard prevents unbounded recursion on malformed trees. + """ + if max_depth <= 0: + return 0 + if isinstance(root, list): + return sum(count_decision_nodes(item, max_depth=max_depth) for item in root) + if isinstance(root, dict): + has_decision = 1 if str(root.get("decision_id", "")).strip() else 0 + children = root.get("children") or [] + return has_decision + sum( + count_decision_nodes(child, max_depth=max_depth - 1) for child in children + ) + return 0 + + +def _build_snapshot(plan_id: str) -> dict[str, Any]: + """Build a serialisable snapshot for WF04 verification assertions.""" + lifecycle_service = get_container().plan_lifecycle_service() + plan = lifecycle_service.get_plan(plan_id) + if plan is None: + msg = f"plan '{plan_id}' not found" + raise ValueError(msg) + + project_scopes: list[dict[str, Any]] = [] + resource_to_project: dict[str, str] = {} + + if plan.multi_project_metadata is not None: + for scope in plan.multi_project_metadata.project_scopes: + project_scopes.append( + { + "project_name": scope.project_name, + "resource_ids": list(scope.resource_ids), + } + ) + for resource_id in scope.resource_ids: + resource_to_project[resource_id] = scope.project_name + + subplans: list[dict[str, Any]] = [] + for status in plan.subplan_statuses: + child_plan = lifecycle_service.get_plan(status.subplan_id) + mapped_projects = sorted( + { + resource_to_project[resource_id] + for resource_id in status.target_resources + if resource_id in resource_to_project + } + ) + unmapped_resources = sorted( + { + resource_id + for resource_id in status.target_resources + if resource_id not in resource_to_project + } + ) + + subplans.append( + { + "subplan_id": status.subplan_id, + "status": _enum_value(status.status), + "started_at": _iso(status.started_at), + "completed_at": _iso(status.completed_at), + "target_resources": list(status.target_resources), + "mapped_projects": mapped_projects, + "unmapped_resources": unmapped_resources, + "files_changed": status.files_changed, + "child_phase": ( + _enum_value(child_plan.phase) if child_plan is not None else "" + ), + "child_state": ( + _enum_value(child_plan.processing_state) + if child_plan is not None + else "" + ), + "child_updated_at": ( + _iso(child_plan.timestamps.updated_at) + if child_plan is not None + else "" + ), + "execute_started_at": ( + _iso(child_plan.timestamps.execute_started_at) + if child_plan is not None + else "" + ), + "execute_completed_at": ( + _iso(child_plan.timestamps.execute_completed_at) + if child_plan is not None + else "" + ), + "apply_started_at": ( + _iso(child_plan.timestamps.apply_started_at) + if child_plan is not None + else "" + ), + "applied_at": ( + _iso(child_plan.timestamps.applied_at) + if child_plan is not None + else "" + ), + "child_validation_summary": ( + child_plan.validation_summary if child_plan is not None else None + ), + } + ) + + return { + "plan_id": plan.identity.plan_id, + "subplan_count": len(plan.subplan_statuses), + "validation_summary": plan.validation_summary, + "project_scopes": project_scopes, + "subplans": subplans, + } + + +def main() -> int: + """CLI entrypoint. + + Modes: + wf04_snapshot_helper.py + Build and print the full snapshot JSON for a plan. + wf04_snapshot_helper.py --count-nodes + Read a JSON tree from *json_file* (or ``-`` for stdin) and + print the decision-node count to stdout. + """ + if len(sys.argv) == 3 and sys.argv[1] == "--count-nodes": + return _cli_count_nodes(sys.argv[2]) + + if len(sys.argv) != 2: + print( + "Usage: wf04_snapshot_helper.py \n" + " wf04_snapshot_helper.py --count-nodes \n", + file=sys.stderr, + ) + return 2 + + plan_id = sys.argv[1] + try: + snapshot = _build_snapshot(plan_id) + except (ValueError, RuntimeError) as exc: + print(str(exc), file=sys.stderr) + return 1 + # Intentional catch-all: CLI entrypoint must never propagate + # unhandled exceptions — report full traceback and exit non-zero. + except Exception as exc: + print(f"{exc}\n{traceback.format_exc()}", file=sys.stderr) + return 1 + + print(json.dumps(snapshot, default=str)) + return 0 + + +def _cli_count_nodes(json_source: str) -> int: + """Read a JSON tree and print the decision-node count.""" + try: + if json_source == "-": + tree = json.load(sys.stdin) + else: + tree = json.loads(Path(json_source).read_text(encoding="utf-8")) + print(count_decision_nodes(tree)) + except (ValueError, OSError, json.JSONDecodeError) as exc: + print(str(exc), file=sys.stderr) + return 1 + # Intentional catch-all: CLI must not propagate exceptions. + except Exception as exc: + print(f"{exc}\n{traceback.format_exc()}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- 2.52.0