Files
cleveragents-core/features/steps/wf04_snapshot_helper_steps.py
freemo 845cf61b47
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 38s
CI / security (pull_request) Successful in 1m17s
CI / build (pull_request) Successful in 25s
CI / helm (pull_request) Successful in 37s
CI / quality (pull_request) Successful in 3m43s
CI / typecheck (pull_request) Successful in 3m58s
CI / integration_tests (pull_request) Successful in 6m4s
CI / unit_tests (pull_request) Successful in 6m22s
CI / docker (pull_request) Successful in 21s
CI / e2e_tests (pull_request) Successful in 9m29s
CI / coverage (pull_request) Successful in 9m12s
CI / status-check (pull_request) Successful in 1s
CI / lint (push) Successful in 31s
CI / security (push) Successful in 55s
CI / build (push) Successful in 21s
CI / helm (push) Successful in 22s
CI / quality (push) Successful in 3m45s
CI / typecheck (push) Successful in 3m56s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 6m46s
CI / integration_tests (push) Successful in 6m25s
CI / docker (push) Successful in 1m20s
CI / e2e_tests (push) Successful in 9m57s
CI / coverage (push) Successful in 10m50s
CI / status-check (push) Successful in 1s
CI / benchmark-publish (push) Successful in 29m50s
CI / benchmark-regression (pull_request) Successful in 56m29s
test(e2e): workflow example 4 — multi-project dependency update (supervised profile)
Implement WF04 E2E test coverage for multi-project dependency update
using the supervised automation profile. A parent plan targets four
projects (common-lib + 3 services), spawns child subplans per project,
executes in dependency order, validates per project, and applies in
dependency order.

Key components:
- Robot test (robot/e2e/wf04_multi_project.robot): full supervised
  workflow exercising plan use, strategize, execute, validate, apply,
  and final status. JSON output parsing via raw_decode for resilience.
  Test-level guard skips the entire test (visible SKIPPED in CI) when
  the LLM produces zero subplans, preventing silent AC-4/5/6/7 bypass.
  Count Decision Nodes keyword delegates to the Python helper to avoid
  duplicating recursive tree-walking logic.
- Snapshot helper (robot/e2e/wf04_snapshot_helper.py): collects
  deterministic parent/subplan metadata from the lifecycle service.
  Timestamps normalised to UTC. Depth-guarded count_decision_nodes()
  function (max_depth=50) for safe reuse from Robot keywords.
  sys.path.append (not insert) to avoid shadowing the standard library.
- Behave unit tests (features/wf04_snapshot_helper.feature + steps):
  17 scenarios covering _iso(), _enum_value(), count_decision_nodes(),
  and _build_snapshot() with mocked lifecycle service. Includes tests
  for no-subplans (verifies plan_id, project_scopes, validation_summary),
  with-subplans (verifies concrete mapped project names and subplan_count),
  child-plan-is-None (verifies empty-string defaults), unmapped resources,
  and exact-match aware-datetime UTC normalisation.

ISSUES CLOSED: #750
2026-03-30 11:47:37 +08:00

646 lines
23 KiB
Python

"""Step definitions for WF04 snapshot helper unit tests."""
from __future__ import annotations
# The snapshot helper lives outside ``src/`` so we import via path manipulation.
import sys
from datetime import UTC, datetime, timedelta, timezone
from enum import Enum
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
_ROBOT_E2E = Path(__file__).resolve().parents[1].parent / "robot" / "e2e"
if str(_ROBOT_E2E) not in sys.path:
sys.path.append(str(_ROBOT_E2E))
from wf04_snapshot_helper import ( # noqa: E402
_build_snapshot,
_enum_value,
_iso,
count_decision_nodes,
)
# ---------------------------------------------------------------------------
# _iso() steps
# ---------------------------------------------------------------------------
@given("a None timestamp value")
def step_given_none_timestamp(context: Context) -> None:
context.iso_input = None
@given('a non-datetime truthy value "{value}"')
def step_given_non_datetime_truthy(context: Context, value: str) -> None:
context.iso_input = value
@given("a naive datetime {iso_str}")
def step_given_naive_datetime(context: Context, iso_str: str) -> None:
context.iso_input = datetime.fromisoformat(iso_str)
# Store for the "both" scenario
if not hasattr(context, "iso_inputs"):
context.iso_inputs = []
context.iso_inputs.append(context.iso_input)
@given("an aware datetime {iso_str} in timezone {tz_str}")
def step_given_aware_datetime(context: Context, iso_str: str, tz_str: str) -> None:
naive = datetime.fromisoformat(iso_str)
if tz_str == "UTC":
tz = UTC
else:
# Parse offset like "UTC+05:00"
sign = 1 if "+" in tz_str else -1
parts = tz_str.replace("UTC", "").lstrip("+-").split(":")
hours = int(parts[0])
minutes = int(parts[1]) if len(parts) > 1 else 0
tz = timezone(timedelta(hours=sign * hours, minutes=sign * minutes))
context.iso_input = naive.replace(tzinfo=tz)
if not hasattr(context, "iso_inputs"):
context.iso_inputs = []
context.iso_inputs.append(context.iso_input)
@when("_iso is called")
def step_when_iso_called(context: Context) -> None:
context.iso_result = _iso(context.iso_input)
@when("_iso is called on both")
def step_when_iso_called_on_both(context: Context) -> None:
context.iso_results = [_iso(v) for v in context.iso_inputs]
@then("the iso result should be an empty string")
def step_then_result_empty(context: Context) -> None:
assert context.iso_result == "", (
f"Expected empty string, got {context.iso_result!r}"
)
@then('the iso result should contain "{fragment}"')
def step_then_result_contains(context: Context, fragment: str) -> None:
assert fragment in context.iso_result, (
f"Expected {fragment!r} in {context.iso_result!r}"
)
@then("both iso results should be identical")
def step_then_both_results_identical(context: Context) -> None:
assert len(context.iso_results) == 2, "Expected exactly 2 results"
assert context.iso_results[0] == context.iso_results[1], (
f"Results differ: {context.iso_results[0]!r} vs {context.iso_results[1]!r}"
)
# ---------------------------------------------------------------------------
# _enum_value() steps
# ---------------------------------------------------------------------------
class _FakeEnum(Enum):
COMPLETED = "completed"
@given('an enum-like object with value "{value}"')
def step_given_enum_like(context: Context, value: str) -> None:
context.enum_input = _FakeEnum(value)
@given('a plain string "{value}"')
def step_given_plain_string(context: Context, value: str) -> None:
context.enum_input = value
@when("_enum_value is called")
def step_when_enum_value_called(context: Context) -> None:
context.enum_result = _enum_value(context.enum_input)
@then('the enum result should be "{expected}"')
def step_then_result_equals(context: Context, expected: str) -> None:
assert context.enum_result == expected, (
f"Expected {expected!r}, got {context.enum_result!r}"
)
# ---------------------------------------------------------------------------
# count_decision_nodes() steps
# ---------------------------------------------------------------------------
@given("an empty list tree")
def step_given_empty_tree(context: Context) -> None:
empty: list[Any] = []
context.tree_input = empty
@given("a tree with one decision node")
def step_given_one_decision_node(context: Context) -> None:
context.tree_input = {"decision_id": "D001", "children": []}
@given("a tree with a root and 2 child decision nodes")
def step_given_nested_tree(context: Context) -> None:
context.tree_input = {
"decision_id": "D001",
"children": [
{"decision_id": "D002", "children": []},
{"decision_id": "D003", "children": []},
],
}
@given("a tree with mixed decision and non-decision nodes")
def step_given_mixed_tree(context: Context) -> None:
context.tree_input = {
"decision_id": "D001",
"children": [
{"decision_id": "", "children": []}, # no decision_id
{"children": []}, # missing decision_id key
{"decision_id": "D004", "children": []},
],
}
@given("a list of 2 independent decision trees")
def step_given_list_of_trees(context: Context) -> None:
context.tree_input = [
{"decision_id": "D001", "children": []},
{"decision_id": "D002", "children": []},
]
@given("a tree with depth {depth:d} and one decision node per level")
def step_given_deep_tree(context: Context, depth: int) -> None:
"""Build a linear chain of decision nodes at the given depth."""
tree: dict[str, Any] = {"decision_id": f"D{depth:03d}", "children": []}
current = tree
for i in range(depth - 1, 0, -1):
child: dict[str, Any] = {"decision_id": f"D{i:03d}", "children": []}
current["children"] = [child]
current = child
context.tree_input = tree
@when("count_decision_nodes is called with max_depth {max_depth:d}")
def step_when_count_decision_nodes_with_depth(context: Context, max_depth: int) -> None:
context.decision_count = count_decision_nodes(
context.tree_input, max_depth=max_depth
)
@when("count_decision_nodes is called")
def step_when_count_decision_nodes(context: Context) -> None:
context.decision_count = count_decision_nodes(context.tree_input)
@then("the decision node count should be {expected:d}")
def step_then_count_equals(context: Context, expected: int) -> None:
assert context.decision_count == expected, (
f"Expected {expected}, got {context.decision_count}"
)
# ---------------------------------------------------------------------------
# _build_snapshot() steps (requires mocking the DI container)
# ---------------------------------------------------------------------------
def _make_mock_plan(
plan_id: str = "01TEST00000000000000000001",
*,
subplan_statuses: list[Any] | None = None,
multi_project_metadata: Any | None = None,
validation_summary: dict[str, Any] | None = None,
) -> MagicMock:
"""Create a mock Plan object for snapshot testing."""
plan = MagicMock()
plan.identity.plan_id = plan_id
plan.subplan_statuses = subplan_statuses or []
plan.multi_project_metadata = multi_project_metadata
plan.validation_summary = validation_summary
plan.phase = _FakeEnum.COMPLETED
plan.processing_state = _FakeEnum.COMPLETED
plan.timestamps.updated_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC)
plan.timestamps.execute_started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC)
plan.timestamps.execute_completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC)
plan.timestamps.apply_started_at = datetime(2026, 3, 15, 11, 45, 0, tzinfo=UTC)
plan.timestamps.applied_at = datetime(2026, 3, 15, 12, 0, 0, tzinfo=UTC)
return plan
def _make_mock_subplan_status(
subplan_id: str,
target_resources: list[str] | None = None,
) -> MagicMock:
status = MagicMock()
status.subplan_id = subplan_id
status.status = _FakeEnum.COMPLETED
status.started_at = datetime(2026, 3, 15, 11, 0, 0, tzinfo=UTC)
status.completed_at = datetime(2026, 3, 15, 11, 30, 0, tzinfo=UTC)
status.target_resources = target_resources or []
status.files_changed = 1
return status
def _make_mock_project_scope(project_name: str, resource_ids: list[str]) -> MagicMock:
scope = MagicMock()
scope.project_name = project_name
scope.resource_ids = resource_ids
return scope
@given("a mocked lifecycle service that returns None for plan lookup")
def step_given_mock_lifecycle_none(context: Context) -> None:
context.mock_lifecycle = MagicMock()
context.mock_lifecycle.get_plan.return_value = None
context.mock_plan_id = "nonexistent"
@given("a mocked lifecycle service with a plan that has no subplans")
def step_given_mock_plan_no_subplans(context: Context) -> None:
plan = _make_mock_plan()
context.mock_lifecycle = MagicMock()
context.mock_lifecycle.get_plan.return_value = plan
context.mock_plan_id = plan.identity.plan_id
@given("a mocked lifecycle service with a plan that has 2 subplans and project scopes")
def step_given_mock_plan_with_subplans(context: Context) -> None:
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
scope2 = _make_mock_project_scope("proj-b", ["res-2"])
metadata = MagicMock()
metadata.project_scopes = [scope1, scope2]
status1 = _make_mock_subplan_status("SUB01", target_resources=["res-1"])
status2 = _make_mock_subplan_status("SUB02", target_resources=["res-2"])
plan = _make_mock_plan(
subplan_statuses=[status1, status2],
multi_project_metadata=metadata,
)
child1 = _make_mock_plan("SUB01")
child1.validation_summary = {"required_passed": 1, "required_failed": 0}
child2 = _make_mock_plan("SUB02")
child2.validation_summary = {"required_passed": 1, "required_failed": 0}
lifecycle = MagicMock()
lifecycle.get_plan.side_effect = lambda pid: {
plan.identity.plan_id: plan,
"SUB01": child1,
"SUB02": child2,
}.get(pid)
context.mock_lifecycle = lifecycle
context.mock_plan_id = plan.identity.plan_id
@given("a mocked lifecycle service with a subplan targeting an unmapped resource")
def step_given_mock_plan_unmapped_resource(context: Context) -> None:
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
metadata = MagicMock()
metadata.project_scopes = [scope1]
# Subplan targets res-1 (mapped) and res-unknown (unmapped)
status1 = _make_mock_subplan_status(
"SUB01", target_resources=["res-1", "res-unknown"]
)
plan = _make_mock_plan(
subplan_statuses=[status1],
multi_project_metadata=metadata,
)
child1 = _make_mock_plan("SUB01")
child1.validation_summary = None
lifecycle = MagicMock()
lifecycle.get_plan.side_effect = lambda pid: {
plan.identity.plan_id: plan,
"SUB01": child1,
}.get(pid)
context.mock_lifecycle = lifecycle
context.mock_plan_id = plan.identity.plan_id
@when('_build_snapshot is called with plan_id "{plan_id}"')
def step_when_build_snapshot_with_id(context: Context, plan_id: str) -> None:
mock_container = MagicMock()
mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle
with patch("wf04_snapshot_helper.get_container", return_value=mock_container):
try:
context.snapshot_result = _build_snapshot(plan_id)
context.snapshot_error = None
except Exception as exc:
context.snapshot_result = None
context.snapshot_error = exc
@when("_build_snapshot is called")
def step_when_build_snapshot(context: Context) -> None:
mock_container = MagicMock()
mock_container.plan_lifecycle_service.return_value = context.mock_lifecycle
with patch("wf04_snapshot_helper.get_container", return_value=mock_container):
try:
context.snapshot_result = _build_snapshot(context.mock_plan_id)
context.snapshot_error = None
except Exception as exc:
context.snapshot_result = None
context.snapshot_error = exc
@then('the snapshot error should be ValueError containing "{fragment}"')
def step_then_valueerror_raised(context: Context, fragment: str) -> None:
assert context.snapshot_error is not None, (
"Expected an exception but none was raised"
)
assert isinstance(context.snapshot_error, ValueError), (
f"Expected ValueError, got {type(context.snapshot_error).__name__}"
)
assert fragment in str(context.snapshot_error), (
f"Expected {fragment!r} in error message: {context.snapshot_error}"
)
@then("the snapshot should have subplan_count {expected:d}")
def step_then_snapshot_subplan_count(context: Context, expected: int) -> None:
assert context.snapshot_result is not None, "Snapshot should not be None"
assert context.snapshot_result["subplan_count"] == expected, (
f"Expected subplan_count={expected}, got {context.snapshot_result['subplan_count']}"
)
@then("the snapshot subplans list should be empty")
def step_then_snapshot_subplans_empty(context: Context) -> None:
assert context.snapshot_result is not None
assert context.snapshot_result["subplans"] == [], (
f"Expected empty subplans, got {context.snapshot_result['subplans']}"
)
@then("each subplan should have mapped_projects populated")
def step_then_subplan_mapped_projects(context: Context) -> None:
assert context.snapshot_result is not None
for sp in context.snapshot_result["subplans"]:
assert "mapped_projects" in sp, "Subplan missing mapped_projects field"
assert len(sp["mapped_projects"]) > 0, (
f"Subplan {sp['subplan_id']} has empty mapped_projects"
)
@then("each subplan should have an unmapped_resources field")
def step_then_subplan_unmapped_field(context: Context) -> None:
assert context.snapshot_result is not None
for sp in context.snapshot_result["subplans"]:
assert "unmapped_resources" in sp, "Subplan missing unmapped_resources field"
@then("the subplan unmapped_resources should contain the unknown resource ID")
def step_then_subplan_unmapped_contains(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) == 1
assert "res-unknown" in subplans[0]["unmapped_resources"], (
f"Expected 'res-unknown' in unmapped_resources: {subplans[0]['unmapped_resources']}"
)
# ---------------------------------------------------------------------------
# Additional _build_snapshot() assertions
# ---------------------------------------------------------------------------
@then("the snapshot plan_id should match the mocked plan ID")
def step_then_snapshot_plan_id_matches(context: Context) -> None:
assert context.snapshot_result is not None
assert context.snapshot_result["plan_id"] == context.mock_plan_id, (
f"Expected plan_id={context.mock_plan_id!r}, "
f"got {context.snapshot_result['plan_id']!r}"
)
@then("the snapshot project_scopes should be an empty list")
def step_then_snapshot_project_scopes_empty(context: Context) -> None:
assert context.snapshot_result is not None
assert context.snapshot_result["project_scopes"] == [], (
f"Expected empty project_scopes, got {context.snapshot_result['project_scopes']}"
)
@then("the snapshot validation_summary should be None")
def step_then_snapshot_validation_summary_none(context: Context) -> None:
assert context.snapshot_result is not None
assert context.snapshot_result["validation_summary"] is None, (
f"Expected None validation_summary, "
f"got {context.snapshot_result['validation_summary']!r}"
)
@then('subplan "{subplan_id}" should map to project "{project_name}"')
def step_then_subplan_maps_to_project(
context: Context, subplan_id: str, project_name: str
) -> None:
assert context.snapshot_result is not None
matching = [
sp
for sp in context.snapshot_result["subplans"]
if sp["subplan_id"] == subplan_id
]
assert len(matching) == 1, (
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
)
assert project_name in matching[0]["mapped_projects"], (
f"Expected {project_name!r} in mapped_projects of {subplan_id}, "
f"got {matching[0]['mapped_projects']}"
)
@then('the iso result should be exactly "{expected}"')
def step_then_iso_result_exact(context: Context, expected: str) -> None:
assert context.iso_result == expected, (
f"Expected exactly {expected!r}, got {context.iso_result!r}"
)
@then('subplan "{subplan_id}" status should be "{expected}"')
def step_then_subplan_status(context: Context, subplan_id: str, expected: str) -> None:
assert context.snapshot_result is not None
matching = [
sp
for sp in context.snapshot_result["subplans"]
if sp["subplan_id"] == subplan_id
]
assert len(matching) == 1, (
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
)
assert matching[0]["status"] == expected, (
f"Expected status={expected!r} for {subplan_id}, got {matching[0]['status']!r}"
)
@then('subplan "{subplan_id}" child_phase should be "{expected}"')
def step_then_subplan_child_phase(
context: Context, subplan_id: str, expected: str
) -> None:
assert context.snapshot_result is not None
matching = [
sp
for sp in context.snapshot_result["subplans"]
if sp["subplan_id"] == subplan_id
]
assert len(matching) == 1, (
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
)
assert matching[0]["child_phase"] == expected, (
f"Expected child_phase={expected!r} for {subplan_id}, "
f"got {matching[0]['child_phase']!r}"
)
@then('subplan "{subplan_id}" started_at should be non-empty')
def step_then_subplan_started_at_non_empty(context: Context, subplan_id: str) -> None:
assert context.snapshot_result is not None
matching = [
sp
for sp in context.snapshot_result["subplans"]
if sp["subplan_id"] == subplan_id
]
assert len(matching) == 1, (
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
)
assert matching[0]["started_at"] != "", (
f"Expected non-empty started_at for {subplan_id}, "
f"got {matching[0]['started_at']!r}"
)
@then(
'subplan "{subplan_id}" child_validation_summary required_passed should be {expected:d}'
)
def step_then_subplan_validation_required_passed(
context: Context, subplan_id: str, expected: int
) -> None:
assert context.snapshot_result is not None
matching = [
sp
for sp in context.snapshot_result["subplans"]
if sp["subplan_id"] == subplan_id
]
assert len(matching) == 1, (
f"Expected exactly one subplan with id {subplan_id!r}, found {len(matching)}"
)
summary = matching[0].get("child_validation_summary")
assert summary is not None, (
f"Expected child_validation_summary for {subplan_id}, got None"
)
actual = int(summary.get("required_passed", 0))
assert actual == expected, (
f"Expected required_passed={expected} for {subplan_id}, got {actual}"
)
# ---------------------------------------------------------------------------
# _build_snapshot() child plan is None scenario
# ---------------------------------------------------------------------------
@given("a mocked lifecycle service with a subplan whose child plan is None")
def step_given_mock_plan_child_none(context: Context) -> None:
scope1 = _make_mock_project_scope("proj-a", ["res-1"])
metadata = MagicMock()
metadata.project_scopes = [scope1]
status1 = _make_mock_subplan_status("SUB_NONE", target_resources=["res-1"])
plan = _make_mock_plan(
subplan_statuses=[status1],
multi_project_metadata=metadata,
)
lifecycle = MagicMock()
# Parent plan found, but get_plan returns None for the child subplan
lifecycle.get_plan.side_effect = lambda pid: (
plan if pid == plan.identity.plan_id else None
)
context.mock_lifecycle = lifecycle
context.mock_plan_id = plan.identity.plan_id
@then("the subplan child_phase should be an empty string")
def step_then_child_phase_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["child_phase"] == "", (
f"Expected empty child_phase, got {subplans[0]['child_phase']!r}"
)
@then("the subplan child_state should be an empty string")
def step_then_child_state_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["child_state"] == "", (
f"Expected empty child_state, got {subplans[0]['child_state']!r}"
)
@then("the subplan child_updated_at should be an empty string")
def step_then_child_updated_at_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["child_updated_at"] == "", (
f"Expected empty child_updated_at, got {subplans[0]['child_updated_at']!r}"
)
@then("the subplan execute_started_at should be an empty string")
def step_then_execute_started_at_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["execute_started_at"] == "", (
f"Expected empty execute_started_at, got {subplans[0]['execute_started_at']!r}"
)
@then("the subplan execute_completed_at should be an empty string")
def step_then_execute_completed_at_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["execute_completed_at"] == "", (
f"Expected empty execute_completed_at, got {subplans[0]['execute_completed_at']!r}"
)
@then("the subplan apply_started_at should be an empty string")
def step_then_apply_started_at_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["apply_started_at"] == "", (
f"Expected empty apply_started_at, got {subplans[0]['apply_started_at']!r}"
)
@then("the subplan applied_at should be an empty string")
def step_then_applied_at_empty(context: Context) -> None:
assert context.snapshot_result is not None
subplans = context.snapshot_result["subplans"]
assert len(subplans) >= 1
assert subplans[0]["applied_at"] == "", (
f"Expected empty applied_at, got {subplans[0]['applied_at']!r}"
)