freemo
48cff5cfe0
CI / build (push) Successful in 18s
CI / lint (push) Failing after 31s
CI / helm (push) Successful in 33s
CI / typecheck (push) Successful in 50s
CI / security (push) Failing after 51s
CI / coverage (push) Has been skipped
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Failing after 1m50s
CI / docker (push) Has been skipped
CI / quality (push) Successful in 3m43s
CI / integration_tests (push) Has been cancelled
CI / e2e_tests (push) Has been cancelled
CI / benchmark-publish (push) Has been cancelled
CI / status-check (push) Has been cancelled
Renames `plan lifecycle-list` to `plan list` and `plan lifecycle-apply` to `plan apply` to align with the specification's canonical command names. Removes legacy V2 plan commands that occupied those names. - Renamed CLI command registrations from lifecycle-list/lifecycle-apply to list/apply - Removed legacy V2 apply and list commands (~200 lines) - Updated apply shortcut in main.py to delegate to v3 lifecycle - Added defensive null check for plan existence in apply command - Updated 63+ test, doc, and benchmark files for consistency Closes #881 Co-authored-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me> Co-committed-by: Jeffrey Phillips Freeman <the@jeffreyfreeman.me>
866 lines
29 KiB
Python
866 lines
29 KiB
Python
"""Robot Framework helper for M6 E2E verification tests.
|
|
|
|
Exercises the complete M6 success criteria sequence:
|
|
1. Porting action creation from YAML config
|
|
2. Plan use + execute via real CLI subprocess
|
|
3. Hierarchical decomposition: 4+ levels of subplans
|
|
4. Decision correction recomputes only affected subtree
|
|
5. Parallel execution scales to 10+ concurrent subplans
|
|
6. Realistic porting task completes autonomously (lifecycle)
|
|
7. Plan apply transitions to APPLIED state via real CLI subprocess
|
|
8. SubplanFailureHandler retry and stop-others logic
|
|
|
|
CLI-facing tests (action-create-porting, plan-use-execute, plan-apply-lifecycle)
|
|
invoke the real ``agents`` CLI via subprocess without mocking.
|
|
Domain-level tests exercise real application code directly.
|
|
|
|
Each subcommand prints a sentinel string on success and exits 0.
|
|
On failure it prints a diagnostic to stderr and exits 1.
|
|
|
|
Usage:
|
|
python robot/helper_m6_e2e_verification.py <command>
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import NoReturn
|
|
|
|
# Ensure src is importable when run from workspace root
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Ensure robot/ is on the import path for helper_e2e_common.
|
|
_ROBOT = str(Path(__file__).resolve().parent)
|
|
if _ROBOT not in sys.path:
|
|
sys.path.insert(0, _ROBOT)
|
|
|
|
from helper_e2e_common import ( # noqa: E402
|
|
cleanup_workspace,
|
|
is_expected_provider_unavailable,
|
|
run_cli,
|
|
setup_workspace,
|
|
write_yaml,
|
|
)
|
|
from helpers_common import reset_global_state # noqa: E402
|
|
from ulid import ULID # noqa: E402
|
|
|
|
from cleveragents.domain.models.core.correction import ( # noqa: E402
|
|
CorrectionDryRunReport,
|
|
CorrectionImpact,
|
|
CorrectionMode,
|
|
CorrectionRequest,
|
|
CorrectionStatus,
|
|
)
|
|
from cleveragents.domain.models.core.decision import ( # noqa: E402
|
|
Decision,
|
|
DecisionType,
|
|
)
|
|
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
|
ExecutionMode,
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanPhase,
|
|
ProcessingState,
|
|
SubplanConfig,
|
|
SubplanFailureHandler,
|
|
SubplanMergeStrategy,
|
|
SubplanStatus,
|
|
)
|
|
|
|
# -------------------------------------------------------------------
|
|
# Valid ULID constants
|
|
# -------------------------------------------------------------------
|
|
_ROOT_PLAN_ID = str(ULID())
|
|
_CHILD_PLAN_IDS = [str(ULID()) for _ in range(4)]
|
|
_GRANDCHILD_PLAN_IDS = [str(ULID()) for _ in range(8)]
|
|
_GREAT_GRANDCHILD_IDS = [str(ULID()) for _ in range(4)]
|
|
_LEAF_IDS = [str(ULID()) for _ in range(4)]
|
|
_ACTION_ULID = str(ULID())
|
|
|
|
_ROOT_DEC_ID = str(ULID())
|
|
_CHILD_DEC_IDS = [str(ULID()) for _ in range(4)]
|
|
|
|
_ACTION_YAML = """\
|
|
name: local/port-to-typescript
|
|
description: Autonomous Python to TypeScript porting action
|
|
definition_of_done: All Python files converted to TypeScript with tests
|
|
strategy_actor: openai/gpt-4
|
|
execution_actor: openai/gpt-4
|
|
"""
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Helpers
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def _fail(msg: str) -> NoReturn:
|
|
"""Print failure message to stderr and exit with code 1."""
|
|
print(f"FAIL: {msg}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def _extract_plan_id(output: str) -> str | None:
|
|
"""Extract a ULID plan_id from plain CLI output."""
|
|
match = re.search(r"\b([0-9A-Z]{26})\b", output)
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _make_plan(
|
|
*,
|
|
plan_id: str = _ROOT_PLAN_ID,
|
|
parent_plan_id: str | None = None,
|
|
root_plan_id: str | None = None,
|
|
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
|
processing_state: ProcessingState = ProcessingState.QUEUED,
|
|
subplan_config: SubplanConfig | None = None,
|
|
subplan_statuses: list[SubplanStatus] | None = None,
|
|
action_name: str = "local/port-to-typescript",
|
|
description: str = "Port Python to TypeScript",
|
|
) -> Plan:
|
|
"""Create a Plan for testing with minimal required fields."""
|
|
return Plan(
|
|
identity=PlanIdentity(
|
|
plan_id=plan_id,
|
|
parent_plan_id=parent_plan_id,
|
|
root_plan_id=root_plan_id,
|
|
),
|
|
namespaced_name=NamespacedName(
|
|
server=None,
|
|
namespace="local",
|
|
name="port-to-typescript",
|
|
),
|
|
description=description,
|
|
action_name=action_name,
|
|
phase=phase,
|
|
processing_state=processing_state,
|
|
subplan_config=subplan_config,
|
|
subplan_statuses=subplan_statuses or [],
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: action-create-porting (CLI via subprocess)
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def action_create_porting() -> None:
|
|
"""Create a porting action from YAML config via real CLI subprocess.
|
|
|
|
Writes a YAML file, invokes ``agents action create --config``
|
|
via subprocess, and verifies the action is persisted by querying
|
|
the database.
|
|
"""
|
|
workspace = setup_workspace(prefix="m6_action_")
|
|
yaml_path = write_yaml(_ACTION_YAML)
|
|
try:
|
|
r = run_cli("action", "create", "--config", yaml_path, workspace=workspace)
|
|
if r.returncode != 0:
|
|
_fail(f"action create: {r.stderr}")
|
|
|
|
combined = r.stdout + r.stderr
|
|
if "INTERNAL" in combined or "Traceback" in combined:
|
|
_fail(f"action create crashed:\n{combined}")
|
|
|
|
# Verify the output references the action name
|
|
if "port-to-typescript" not in combined:
|
|
_fail(f"action name not in output:\n{combined}")
|
|
|
|
print("m6-action-create-porting-ok")
|
|
finally:
|
|
os.unlink(yaml_path)
|
|
cleanup_workspace(workspace)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: plan-use-execute (CLI via subprocess)
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def plan_use_execute() -> None:
|
|
"""Use and execute a porting plan via real CLI subprocess.
|
|
|
|
Creates an action, then runs ``plan use`` and ``plan execute``
|
|
via subprocess. The plan is in strategize/queued so execute
|
|
correctly rejects with a controlled "not ready" message.
|
|
"""
|
|
workspace = setup_workspace(prefix="m6_use_exec_")
|
|
yaml_path = write_yaml(_ACTION_YAML)
|
|
try:
|
|
# Create the action first
|
|
r1 = run_cli("action", "create", "--config", yaml_path, workspace=workspace)
|
|
if r1.returncode != 0:
|
|
_fail(f"action create: {r1.stderr}")
|
|
|
|
# Plan use
|
|
r2 = run_cli(
|
|
"plan",
|
|
"use",
|
|
"local/port-to-typescript",
|
|
"local/large-project",
|
|
"--format",
|
|
"plain",
|
|
workspace=workspace,
|
|
)
|
|
if r2.returncode != 0:
|
|
_fail(f"plan use: {r2.stderr}")
|
|
|
|
plan_id = _extract_plan_id(r2.stdout)
|
|
if not plan_id:
|
|
_fail(f"could not extract plan_id:\n{r2.stdout}")
|
|
|
|
# Plan execute — plan is in strategize/queued, so this will
|
|
# get a controlled rejection (not a crash).
|
|
r3 = run_cli("plan", "execute", plan_id, workspace=workspace)
|
|
combined = r3.stdout + r3.stderr
|
|
if "Traceback" in combined:
|
|
_fail(f"plan execute crashed:\n{combined}")
|
|
if "INTERNAL" in combined and not is_expected_provider_unavailable(combined):
|
|
_fail(f"plan execute crashed:\n{combined}")
|
|
# Verify the output references the plan
|
|
if plan_id not in combined:
|
|
_fail(f"plan execute output missing plan_id {plan_id}")
|
|
|
|
print("m6-plan-use-execute-ok")
|
|
finally:
|
|
os.unlink(yaml_path)
|
|
cleanup_workspace(workspace)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: hierarchical-decomposition
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def hierarchical_decomposition() -> None:
|
|
"""Verify hierarchical decomposition creates 4+ levels of subplans.
|
|
|
|
Constructs a tree: root → L1 (4 children) → L2 (8 grandchildren)
|
|
→ L3 (4 great-grandchildren) → L4 (4 leaves).
|
|
Verifies identity flags and root_plan_id propagation.
|
|
"""
|
|
# Level 0: root plan
|
|
root = _make_plan(plan_id=_ROOT_PLAN_ID, root_plan_id=_ROOT_PLAN_ID)
|
|
if not root.is_root_plan:
|
|
_fail("root should be is_root_plan")
|
|
if root.is_subplan:
|
|
_fail("root should not be is_subplan")
|
|
|
|
# Level 1: 4 children
|
|
level1 = []
|
|
for cid in _CHILD_PLAN_IDS:
|
|
p = _make_plan(
|
|
plan_id=cid,
|
|
parent_plan_id=_ROOT_PLAN_ID,
|
|
root_plan_id=_ROOT_PLAN_ID,
|
|
description=f"L1 subplan {cid[:8]}",
|
|
)
|
|
if not p.is_subplan:
|
|
_fail(f"L1 {cid} should be subplan")
|
|
if p.identity.root_plan_id != _ROOT_PLAN_ID:
|
|
_fail(f"L1 root_plan_id mismatch: {p.identity.root_plan_id}")
|
|
level1.append(p)
|
|
|
|
if len(level1) < 4:
|
|
_fail(f"expected >= 4 L1 subplans, got {len(level1)}")
|
|
|
|
# Level 2: 8 grandchildren (2 per L1)
|
|
level2 = []
|
|
for i, gcid in enumerate(_GRANDCHILD_PLAN_IDS):
|
|
parent = _CHILD_PLAN_IDS[i % len(_CHILD_PLAN_IDS)]
|
|
p = _make_plan(
|
|
plan_id=gcid,
|
|
parent_plan_id=parent,
|
|
root_plan_id=_ROOT_PLAN_ID,
|
|
description=f"L2 subplan {gcid[:8]}",
|
|
)
|
|
if p.identity.root_plan_id != _ROOT_PLAN_ID:
|
|
_fail(f"L2 root mismatch: {p.identity.root_plan_id}")
|
|
level2.append(p)
|
|
|
|
if len(level2) < 4:
|
|
_fail(f"expected >= 4 L2 subplans, got {len(level2)}")
|
|
|
|
# Level 3: 4 great-grandchildren
|
|
level3 = []
|
|
for i, ggid in enumerate(_GREAT_GRANDCHILD_IDS):
|
|
parent = _GRANDCHILD_PLAN_IDS[i % len(_GRANDCHILD_PLAN_IDS)]
|
|
p = _make_plan(
|
|
plan_id=ggid,
|
|
parent_plan_id=parent,
|
|
root_plan_id=_ROOT_PLAN_ID,
|
|
description=f"L3 subplan {ggid[:8]}",
|
|
)
|
|
level3.append(p)
|
|
|
|
# Level 4: 4 leaves
|
|
level4 = []
|
|
for i, lid in enumerate(_LEAF_IDS):
|
|
parent = _GREAT_GRANDCHILD_IDS[i % len(_GREAT_GRANDCHILD_IDS)]
|
|
p = _make_plan(
|
|
plan_id=lid,
|
|
parent_plan_id=parent,
|
|
root_plan_id=_ROOT_PLAN_ID,
|
|
description=f"L4 leaf subplan {lid[:8]}",
|
|
)
|
|
level4.append(p)
|
|
|
|
total = 1 + len(level1) + len(level2) + len(level3) + len(level4)
|
|
levels = 5 # L0, L1, L2, L3, L4
|
|
if levels < 5:
|
|
_fail(f"expected >= 5 levels (4+ subplan levels), got {levels}")
|
|
|
|
# Verify all non-root plans propagate root_plan_id
|
|
for plan in [*level1, *level2, *level3, *level4]:
|
|
if plan.identity.root_plan_id != _ROOT_PLAN_ID:
|
|
_fail(f"root_plan_id not propagated: {plan.identity.plan_id}")
|
|
if not plan.is_subplan:
|
|
_fail(f"should be subplan: {plan.identity.plan_id}")
|
|
|
|
if total < 21:
|
|
_fail(f"expected >= 21 total plans, got {total}")
|
|
|
|
print("m6-hierarchical-decomposition-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: correction-affected-subtree
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def correction_affected_subtree() -> None:
|
|
"""Verify decision correction recomputes only affected subtree.
|
|
|
|
Creates a decision tree (root → 4 children), then simulates
|
|
correction targeting one child. Verifies CorrectionImpact
|
|
contains only the affected subtree, not the entire tree.
|
|
"""
|
|
# Root decision
|
|
root = Decision(
|
|
decision_id=_ROOT_DEC_ID,
|
|
plan_id=_ROOT_PLAN_ID,
|
|
parent_decision_id=None,
|
|
sequence_number=0,
|
|
decision_type=DecisionType.PROMPT_DEFINITION,
|
|
question="How to port this project?",
|
|
chosen_option="Module-by-module porting strategy",
|
|
rationale="Systematic approach for large codebase",
|
|
downstream_decision_ids=_CHILD_DEC_IDS,
|
|
)
|
|
|
|
if root.parent_decision_id is not None:
|
|
_fail("root decision should have no parent")
|
|
if len(root.downstream_decision_ids) != len(_CHILD_DEC_IDS):
|
|
_fail(f"root downstream count: {len(root.downstream_decision_ids)}")
|
|
|
|
# 4 child decisions (one per module)
|
|
children = []
|
|
for i, cid in enumerate(_CHILD_DEC_IDS):
|
|
d = Decision(
|
|
decision_id=cid,
|
|
plan_id=_ROOT_PLAN_ID,
|
|
parent_decision_id=_ROOT_DEC_ID,
|
|
sequence_number=i + 1,
|
|
decision_type=DecisionType.SUBPLAN_SPAWN,
|
|
question=f"How to port module {i}?",
|
|
chosen_option=f"Direct translation for module {i}",
|
|
rationale=f"Module {i} has 1:1 Python→TS mapping",
|
|
)
|
|
children.append(d)
|
|
|
|
# Target child[1] for correction
|
|
target_id = _CHILD_DEC_IDS[1]
|
|
|
|
# Build CorrectionImpact scoped to just the target subtree
|
|
impact = CorrectionImpact(
|
|
affected_decisions=[target_id],
|
|
affected_files=[f"src/module_1/file_{j}.py" for j in range(5)],
|
|
affected_child_plans=[],
|
|
risk_level="low",
|
|
rollback_tier="phase",
|
|
artifacts_to_archive=[],
|
|
)
|
|
|
|
# Verify only 1 decision affected (not the root or other children)
|
|
if len(impact.affected_decisions) != 1:
|
|
_fail(f"expected 1 affected decision, got {len(impact.affected_decisions)}")
|
|
if impact.affected_decisions[0] != target_id:
|
|
_fail(f"wrong affected decision: {impact.affected_decisions[0]}")
|
|
|
|
# Verify root is NOT affected
|
|
if _ROOT_DEC_ID in impact.affected_decisions:
|
|
_fail("root should not be in affected decisions")
|
|
|
|
# Verify other children not affected
|
|
for i, cid in enumerate(_CHILD_DEC_IDS):
|
|
if i == 1:
|
|
continue
|
|
if cid in impact.affected_decisions:
|
|
_fail(f"child {i} should not be affected")
|
|
|
|
# Verify rollback_tier supports subtree-only
|
|
if impact.rollback_tier not in ("full", "phase", "append_only"):
|
|
_fail(f"unexpected rollback_tier: {impact.rollback_tier}")
|
|
|
|
# Build dry-run report
|
|
report = CorrectionDryRunReport(
|
|
correction_id=str(ULID()),
|
|
mode=CorrectionMode.REVERT,
|
|
impact=impact,
|
|
decisions_to_invalidate=[target_id],
|
|
)
|
|
|
|
if len(report.decisions_to_invalidate) != 1:
|
|
_fail(
|
|
f"expected 1 decision to invalidate, "
|
|
f"got {len(report.decisions_to_invalidate)}"
|
|
)
|
|
|
|
# Verify CorrectionRequest
|
|
request = CorrectionRequest(
|
|
plan_id=_ROOT_PLAN_ID,
|
|
target_decision_id=target_id,
|
|
mode=CorrectionMode.REVERT,
|
|
guidance="Re-evaluate module 1 porting strategy",
|
|
dry_run=True,
|
|
)
|
|
if request.status != CorrectionStatus.PENDING:
|
|
_fail(f"initial status should be PENDING: {request.status}")
|
|
|
|
print("m6-correction-affected-subtree-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: parallel-execution-scale
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def parallel_execution_scale() -> None:
|
|
"""Verify parallel execution scales to 10+ concurrent subplans.
|
|
|
|
Constructs a SubplanConfig with PARALLEL mode and max_parallel=15,
|
|
creates 15 SubplanStatuses, and verifies the plan model handles
|
|
them correctly.
|
|
"""
|
|
config = SubplanConfig(
|
|
execution_mode=ExecutionMode.PARALLEL,
|
|
merge_strategy=SubplanMergeStrategy.GIT_THREE_WAY,
|
|
max_parallel=15,
|
|
fail_fast=False,
|
|
retry_failed=True,
|
|
max_retries=2,
|
|
)
|
|
|
|
if config.execution_mode != ExecutionMode.PARALLEL:
|
|
_fail(f"execution_mode mismatch: {config.execution_mode}")
|
|
if config.max_parallel != 15:
|
|
_fail(f"max_parallel mismatch: {config.max_parallel}")
|
|
|
|
# Create 15 subplan statuses (simulating 15 concurrent subplans)
|
|
statuses: list[SubplanStatus] = []
|
|
for i in range(15):
|
|
st = SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/port-module",
|
|
target_resources=[f"src/module_{i:02d}/"],
|
|
status=ProcessingState.PROCESSING,
|
|
)
|
|
statuses.append(st)
|
|
|
|
if len(statuses) < 10:
|
|
_fail(f"expected >= 10 concurrent subplans, got {len(statuses)}")
|
|
|
|
# Create plan with parallel config
|
|
plan = _make_plan(
|
|
subplan_config=config,
|
|
subplan_statuses=statuses,
|
|
phase=PlanPhase.EXECUTE,
|
|
processing_state=ProcessingState.PROCESSING,
|
|
)
|
|
|
|
if not plan.has_subplans:
|
|
_fail("plan should have subplans")
|
|
if len(plan.subplan_statuses) != 15:
|
|
_fail(f"expected 15 subplans, got {len(plan.subplan_statuses)}")
|
|
if plan.subplan_config is None:
|
|
_fail("subplan_config is None")
|
|
if plan.subplan_config.max_parallel < 10:
|
|
_fail(f"max_parallel should be >= 10: {plan.subplan_config.max_parallel}")
|
|
|
|
# Verify CLI dict includes subplan info
|
|
cli = plan.as_cli_dict()
|
|
if "subplan_count" not in cli:
|
|
_fail("as_cli_dict missing subplan_count")
|
|
if cli["subplan_count"] != 15:
|
|
_fail(f"subplan_count mismatch: {cli['subplan_count']}")
|
|
|
|
print("m6-parallel-execution-scale-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: porting-task-autonomous
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def porting_task_autonomous() -> None:
|
|
"""Verify a realistic porting task completes autonomously.
|
|
|
|
Simulates the full lifecycle: ACTION → STRATEGIZE → EXECUTE →
|
|
APPLY with subplan decomposition and completion.
|
|
"""
|
|
# Phase 1: ACTION → STRATEGIZE (ACTION can always transition)
|
|
plan_action = _make_plan(
|
|
phase=PlanPhase.ACTION,
|
|
processing_state=ProcessingState.QUEUED,
|
|
)
|
|
if not plan_action.can_transition_to_next_phase:
|
|
_fail("ACTION/QUEUED should allow transition")
|
|
|
|
# Phase 2: STRATEGIZE with subplan config
|
|
plan_strat = _make_plan(
|
|
phase=PlanPhase.STRATEGIZE,
|
|
processing_state=ProcessingState.COMPLETE,
|
|
subplan_config=SubplanConfig(
|
|
execution_mode=ExecutionMode.PARALLEL,
|
|
max_parallel=10,
|
|
),
|
|
)
|
|
if not plan_strat.can_transition_to_next_phase:
|
|
_fail("STRATEGIZE/COMPLETE should allow transition")
|
|
|
|
# Phase 3: EXECUTE with 10 subplans all COMPLETE
|
|
exec_statuses = []
|
|
for i in range(10):
|
|
exec_statuses.append(
|
|
SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/port-module",
|
|
target_resources=[f"src/module_{i}/"],
|
|
status=ProcessingState.COMPLETE,
|
|
started_at=datetime.now(tz=UTC),
|
|
completed_at=datetime.now(tz=UTC),
|
|
files_changed=i * 10 + 5,
|
|
)
|
|
)
|
|
plan_exec = _make_plan(
|
|
phase=PlanPhase.EXECUTE,
|
|
processing_state=ProcessingState.COMPLETE,
|
|
subplan_config=SubplanConfig(
|
|
execution_mode=ExecutionMode.PARALLEL,
|
|
max_parallel=10,
|
|
),
|
|
subplan_statuses=exec_statuses,
|
|
)
|
|
if not plan_exec.can_transition_to_next_phase:
|
|
_fail("EXECUTE/COMPLETE should allow transition")
|
|
if not plan_exec.has_subplans:
|
|
_fail("execute plan should have subplans")
|
|
|
|
# Verify all subplans completed
|
|
for st in plan_exec.subplan_statuses:
|
|
if st.status != ProcessingState.COMPLETE:
|
|
_fail(f"subplan {st.subplan_id} not complete: {st.status}")
|
|
|
|
# Phase 4: APPLY
|
|
plan_apply = _make_plan(
|
|
phase=PlanPhase.APPLY,
|
|
processing_state=ProcessingState.APPLIED,
|
|
subplan_config=plan_exec.subplan_config,
|
|
subplan_statuses=exec_statuses,
|
|
)
|
|
if not plan_apply.is_terminal:
|
|
_fail("APPLIED should be terminal")
|
|
|
|
# Verify total files changed across all subplans
|
|
total_files = sum(st.files_changed for st in plan_apply.subplan_statuses)
|
|
if total_files < 50:
|
|
_fail(f"expected >= 50 files changed, got {total_files}")
|
|
|
|
print("m6-porting-task-autonomous-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: plan-apply-lifecycle
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def plan_apply_lifecycle() -> None:
|
|
"""Verify ``agents plan apply`` via real CLI subprocess.
|
|
|
|
Creates an action and plan via subprocess, then runs
|
|
``plan apply``. The plan is in strategize/queued
|
|
(not execute/complete) so apply correctly rejects
|
|
with a controlled error message.
|
|
"""
|
|
workspace = setup_workspace(prefix="m6_apply_")
|
|
yaml_path = write_yaml(_ACTION_YAML)
|
|
try:
|
|
r1 = run_cli("action", "create", "--config", yaml_path, workspace=workspace)
|
|
if r1.returncode != 0:
|
|
_fail(f"action create: {r1.stderr}")
|
|
|
|
r2 = run_cli(
|
|
"plan",
|
|
"use",
|
|
"local/port-to-typescript",
|
|
"--format",
|
|
"plain",
|
|
workspace=workspace,
|
|
)
|
|
if r2.returncode != 0:
|
|
_fail(f"plan use: {r2.stderr}")
|
|
plan_id = _extract_plan_id(r2.stdout)
|
|
if not plan_id:
|
|
_fail(f"could not extract plan_id:\n{r2.stdout}")
|
|
|
|
# apply on a strategize/queued plan — should get
|
|
# a controlled rejection, not a crash.
|
|
r3 = run_cli("plan", "apply", "--yes", plan_id, workspace=workspace)
|
|
combined = r3.stdout + r3.stderr
|
|
if "INTERNAL" in combined or "Traceback" in combined:
|
|
_fail(f"apply crashed:\n{combined}")
|
|
|
|
print("m6-plan-apply-lifecycle-ok")
|
|
finally:
|
|
os.unlink(yaml_path)
|
|
cleanup_workspace(workspace)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: failure-handler-logic
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def failure_handler_logic() -> None:
|
|
"""Verify SubplanFailureHandler retry and stop-others logic.
|
|
|
|
Tests:
|
|
- fail_fast=True stops others
|
|
- SEQUENTIAL mode stops others
|
|
- Retriable errors are retried
|
|
- Non-retriable errors are not retried
|
|
- Max retries exceeded stops retry
|
|
"""
|
|
handler = SubplanFailureHandler()
|
|
|
|
# fail_fast stops others
|
|
config_ff = SubplanConfig(
|
|
execution_mode=ExecutionMode.PARALLEL,
|
|
fail_fast=True,
|
|
)
|
|
errored = SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/sub",
|
|
status=ProcessingState.ERRORED,
|
|
error="TimeoutError: timed out",
|
|
)
|
|
if not handler.should_stop_others(config_ff, errored):
|
|
_fail("fail_fast=True should stop others")
|
|
|
|
# SEQUENTIAL stops others regardless of fail_fast
|
|
config_seq = SubplanConfig(
|
|
execution_mode=ExecutionMode.SEQUENTIAL,
|
|
fail_fast=False,
|
|
)
|
|
if not handler.should_stop_others(config_seq, errored):
|
|
_fail("SEQUENTIAL should stop others")
|
|
|
|
# Retriable error should retry
|
|
config_retry = SubplanConfig(retry_failed=True, max_retries=2)
|
|
retriable = SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/sub",
|
|
status=ProcessingState.ERRORED,
|
|
error="TimeoutError: timed out",
|
|
attempt_number=1,
|
|
)
|
|
if not handler.should_retry(config_retry, retriable):
|
|
_fail("retriable error should be retried")
|
|
|
|
# Non-retriable error should NOT retry
|
|
non_retriable = SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/sub",
|
|
status=ProcessingState.ERRORED,
|
|
error="ConfigurationError: bad config",
|
|
attempt_number=1,
|
|
)
|
|
if handler.should_retry(config_retry, non_retriable):
|
|
_fail("non-retriable error should not be retried")
|
|
|
|
# Max retries exceeded
|
|
exhausted = SubplanStatus(
|
|
subplan_id=str(ULID()),
|
|
action_name="local/sub",
|
|
status=ProcessingState.ERRORED,
|
|
error="TimeoutError: timed out",
|
|
attempt_number=3,
|
|
)
|
|
if handler.should_retry(config_retry, exhausted):
|
|
_fail("max retries exceeded should not retry")
|
|
|
|
print("m6-failure-handler-logic-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: subplan-config-modes
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def subplan_config_modes() -> None:
|
|
"""Verify SubplanConfig supports all execution modes and merge strategies."""
|
|
# All execution modes
|
|
for mode in ExecutionMode:
|
|
cfg = SubplanConfig(execution_mode=mode)
|
|
if cfg.execution_mode != mode:
|
|
_fail(f"execution_mode mismatch: {cfg.execution_mode} != {mode}")
|
|
|
|
# All merge strategies
|
|
for strat in SubplanMergeStrategy:
|
|
cfg = SubplanConfig(merge_strategy=strat)
|
|
if cfg.merge_strategy != strat:
|
|
_fail(f"merge_strategy mismatch: {cfg.merge_strategy} != {strat}")
|
|
|
|
# max_parallel bounds
|
|
cfg_min = SubplanConfig(max_parallel=1)
|
|
if cfg_min.max_parallel != 1:
|
|
_fail(f"max_parallel=1 mismatch: {cfg_min.max_parallel}")
|
|
|
|
cfg_max = SubplanConfig(max_parallel=50)
|
|
if cfg_max.max_parallel != 50:
|
|
_fail(f"max_parallel=50 mismatch: {cfg_max.max_parallel}")
|
|
|
|
# Timeout
|
|
cfg_timeout = SubplanConfig(timeout_per_subplan_seconds=600)
|
|
if cfg_timeout.timeout_per_subplan_seconds != 600:
|
|
_fail(f"timeout mismatch: {cfg_timeout.timeout_per_subplan_seconds}")
|
|
|
|
# Default values
|
|
default = SubplanConfig()
|
|
if default.execution_mode != ExecutionMode.SEQUENTIAL:
|
|
_fail(f"default mode: {default.execution_mode}")
|
|
if default.merge_strategy != SubplanMergeStrategy.GIT_THREE_WAY:
|
|
_fail(f"default merge: {default.merge_strategy}")
|
|
if default.max_parallel != 5:
|
|
_fail(f"default max_parallel: {default.max_parallel}")
|
|
|
|
print("m6-subplan-config-modes-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Subcommand: decision-tree-porting
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def decision_tree_porting() -> None:
|
|
"""Verify decision tree structure for a porting task.
|
|
|
|
Creates a decision tree with PROMPT_DEFINITION root,
|
|
SUBPLAN_PARALLEL_SPAWN decisions for module decomposition,
|
|
and verifies parent-child relationships.
|
|
"""
|
|
root = Decision(
|
|
decision_id=_ROOT_DEC_ID,
|
|
plan_id=_ROOT_PLAN_ID,
|
|
parent_decision_id=None,
|
|
sequence_number=0,
|
|
decision_type=DecisionType.PROMPT_DEFINITION,
|
|
question="Port Python project to TypeScript",
|
|
chosen_option="Hierarchical module-by-module porting",
|
|
rationale="Break down into parallel workstreams",
|
|
downstream_decision_ids=_CHILD_DEC_IDS,
|
|
)
|
|
|
|
# PROMPT_DEFINITION must be root (no parent)
|
|
if root.parent_decision_id is not None:
|
|
_fail("PROMPT_DEFINITION should have no parent")
|
|
|
|
# Spawn decisions for each module
|
|
spawn_decisions = []
|
|
for i, cid in enumerate(_CHILD_DEC_IDS):
|
|
d = Decision(
|
|
decision_id=cid,
|
|
plan_id=_ROOT_PLAN_ID,
|
|
parent_decision_id=_ROOT_DEC_ID,
|
|
sequence_number=i + 1,
|
|
decision_type=DecisionType.SUBPLAN_PARALLEL_SPAWN,
|
|
question=f"How to port module_{i:02d}?",
|
|
chosen_option=f"Create subplan for module_{i:02d}",
|
|
rationale=f"Module {i} is independent",
|
|
downstream_plan_ids=[_CHILD_PLAN_IDS[i]],
|
|
)
|
|
spawn_decisions.append(d)
|
|
|
|
# Verify parent-child links
|
|
for d in spawn_decisions:
|
|
if d.parent_decision_id != _ROOT_DEC_ID:
|
|
_fail(f"child {d.decision_id} parent mismatch")
|
|
if d.decision_type != DecisionType.SUBPLAN_PARALLEL_SPAWN:
|
|
_fail(f"child type should be SUBPLAN_PARALLEL_SPAWN: {d.decision_type}")
|
|
if len(d.downstream_plan_ids) != 1:
|
|
_fail(f"child should spawn 1 plan: {len(d.downstream_plan_ids)}")
|
|
|
|
# Root should reference all children
|
|
if len(root.downstream_decision_ids) != len(_CHILD_DEC_IDS):
|
|
_fail(
|
|
f"root downstream count: "
|
|
f"{len(root.downstream_decision_ids)} != {len(_CHILD_DEC_IDS)}"
|
|
)
|
|
|
|
# Verify superseded_by for correction flow
|
|
corrected_root = root.with_superseded_by(str(ULID()))
|
|
if corrected_root.superseded_by is None:
|
|
_fail("superseded_by should be set")
|
|
# Original should be untouched (frozen model)
|
|
if root.superseded_by is not None:
|
|
_fail("original root superseded_by should still be None")
|
|
|
|
print("m6-decision-tree-porting-ok")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Dispatcher
|
|
# -------------------------------------------------------------------
|
|
|
|
_COMMANDS: dict[str, Callable[[], None]] = {
|
|
"action-create-porting": action_create_porting,
|
|
"plan-use-execute": plan_use_execute,
|
|
"hierarchical-decomposition": hierarchical_decomposition,
|
|
"correction-affected-subtree": correction_affected_subtree,
|
|
"parallel-execution-scale": parallel_execution_scale,
|
|
"porting-task-autonomous": porting_task_autonomous,
|
|
"plan-apply-lifecycle": plan_apply_lifecycle,
|
|
"failure-handler-logic": failure_handler_logic,
|
|
"subplan-config-modes": subplan_config_modes,
|
|
"decision-tree-porting": decision_tree_porting,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
"""Entry point called by Robot Framework ``Run Process``."""
|
|
if len(sys.argv) < 2:
|
|
print(
|
|
f"Usage: helper_m6_e2e_verification.py <{'|'.join(_COMMANDS)}>",
|
|
)
|
|
return 1
|
|
command = sys.argv[1]
|
|
handler = _COMMANDS.get(command)
|
|
if handler is None:
|
|
print(f"Unknown command: {command}")
|
|
return 1
|
|
reset_global_state()
|
|
handler()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|