cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

test(e2e): validate M4 acceptance criteria for v3.3.0 milestone closure #560

Merged
hurui200320 merged 1 commits from test/m4-acceptance-gate into master 2026-03-06 03:22:45 +00:00
Conversation 3 Commits 1 Files Changed 5
+294 -2
5 changed files with 294 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+7
View File
@@ -2,6 +2,13 @@
## Unreleased
- Validated M4 acceptance criteria for v3.3.0 milestone closure. All M4 E2E verification
tests and correction/subplan smoke tests pass against the final v3.3.0 implementation.
Added CLI-exercising integration tests for `plan use`, `plan execute`, and `plan tree`
commands to verify the milestone success criteria through actual Typer CLI invocations.
Fixed CONTRIBUTORS.md alphabetical ordering. Full nox quality gates pass: lint, format,
typecheck, unit tests, integration tests, coverage (97%), security scan, dead code
detection, docs build, wheel build, and ASV benchmarks. (#495)
- Validated M3 acceptance criteria for v3.2.0 milestone closure. All 10 E2E
verification tests pass against the final implementation, exercising real
CLI command paths (``plan use``, ``plan execute``, ``plan tree``,
CONTRIBUTORS.md
+1
View File
@@ -2,6 +2,7 @@
* Jeffrey Phillips Freeman <jeffrey.freeman@syncleus.com>
* Brent E. Edwards <brent.edwards@cleverthis.com>
* Rui Hu <rui.hu@cleverthis.com>
* Hamza Khyari <hamza.khyari@cleverthis.com>
* Luis Mendes <luis.p.mendes@gmail.com>
* Rui Hu <rui.hu@cleverthis.com>
features/steps/repositories_uncovered_branches_steps.py
+2 -2
View File
@@ -20,7 +20,7 @@ from types import SimpleNamespace
from behave import given, then, when
from behave.runner import Context
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session, sessionmaker
from cleveragents.domain.models.core import Actor
from cleveragents.infrastructure.database.models import Base
@@ -441,7 +441,7 @@ def step_assert_swapped(context: Context):
@given("repo branch cov an in-memory database with automation profile tables")
def step_automation_db(context: Context):
context.rb_engine = _make_engine()
context.rb_session_factory = sessionmaker(bind=context.rb_engine)
context.rb_session_factory = scoped_session(sessionmaker(bind=context.rb_engine))
@given('repo branch cov an existing automation profile "{name}" with schema "{ver}"')
robot/helper_m4_e2e_verification.py
+253
View File
@@ -8,6 +8,9 @@ Exercises the M4 success criteria for subplans and parallel execution:
5. Three-way merge combines non-conflicting changes
6. Merge conflicts are surfaced correctly
7. Parent plan tracks all subplan statuses
8. CLI plan use creates plan with subplan config
9. CLI plan execute transitions plan with subplans
10. CLI plan tree displays subplan hierarchy
Each subcommand prints a sentinel string on success and exits 0.
On failure it prints a diagnostic to stderr and exits 1.
@@ -18,6 +21,7 @@ Usage:
from __future__ import annotations
import json
import sys
from collections.abc import Callable
from datetime import datetime
@@ -33,6 +37,10 @@ if _SRC not in sys.path:
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.domain.models.core.decision import ( # noqa: E402
Decision,
DecisionType,
)
from cleveragents.domain.models.core.plan import ( # noqa: E402
ExecutionMode,
NamespacedName,
@@ -699,6 +707,248 @@ def parent_tracking() -> None:
print("m4-parent-tracking-ok")
# ---------------------------------------------------------------------------
# Subcommand: cli-plan-use
# ---------------------------------------------------------------------------
def cli_plan_use() -> None:
"""Verify ``agents plan use`` CLI creates a plan with subplan config.
Mocks the lifecycle service and invokes the actual CLI command via
Typer's CliRunner. Asserts the service receives the correct action
name and project, and the CLI exits cleanly.
"""
mock_service = MagicMock()
# Mock action lookup — use_action needs get_action_by_name first
mock_action = MagicMock()
mock_action.namespaced_name = NamespacedName.parse("local/refactor-action")
mock_service.get_action_by_name.return_value = mock_action
# Mock use_action to return a plan in Strategize phase
plan = _mock_parent_plan(
phase=PlanPhase.STRATEGIZE,
state=ProcessingState.QUEUED,
)
mock_service.use_action.return_value = plan
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_service,
):
result = runner.invoke(
plan_app,
["use", "local/refactor-action", "local/monorepo"],
)
if result.exit_code != 0:
_fail(
f"plan use rc={result.exit_code}\n"
f"output={result.output}\n"
f"exception={result.exception}"
)
# Verify the action was looked up
mock_service.get_action_by_name.assert_called_once_with(
"local/refactor-action",
)
# Verify use_action was called with correct project link
mock_service.use_action.assert_called_once()
call_kwargs = mock_service.use_action.call_args
action_arg = call_kwargs.kwargs.get(
"action_name",
call_kwargs[1].get("action_name") if len(call_kwargs) > 1 else None,
)
if action_arg is None and call_kwargs.args:
action_arg = call_kwargs.args[0]
if action_arg != "local/refactor-action":
_fail(f"use_action called with action={action_arg}")
print("m4-cli-plan-use-ok")
# ---------------------------------------------------------------------------
# Subcommand: cli-plan-execute
# ---------------------------------------------------------------------------
def cli_plan_execute() -> None:
"""Verify ``agents plan execute`` CLI transitions plan to Execute.
Mocks the lifecycle service and invokes the actual CLI command.
Asserts the plan transitions to Execute phase and the CLI renders
the plan details.
"""
mock_service = MagicMock()
# get_plan is called for the read-only guard
pre_plan = _mock_parent_plan(
phase=PlanPhase.STRATEGIZE,
state=ProcessingState.COMPLETE,
)
pre_plan.read_only = False
mock_service.get_plan.return_value = pre_plan
# execute_plan returns the plan in Execute phase with subplans
statuses = _make_subplan_statuses()
post_plan = _mock_parent_plan(
phase=PlanPhase.EXECUTE,
state=ProcessingState.PROCESSING,
subplan_statuses=statuses,
)
mock_service.execute_plan.return_value = post_plan
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_service,
):
result = runner.invoke(plan_app, ["execute", _ROOT_ULID])
if result.exit_code != 0:
_fail(
f"plan execute rc={result.exit_code}\n"
f"output={result.output}\n"
f"exception={result.exception}"
)
# Verify execute_plan was called with the plan ID
mock_service.execute_plan.assert_called_once_with(_ROOT_ULID)
# Verify CLI output references the plan (rich rendering)
if _ROOT_ULID not in result.output:
_fail(
f"CLI output should contain plan ID {_ROOT_ULID}\n"
f"output={result.output}"
)
print("m4-cli-plan-execute-ok")
# ---------------------------------------------------------------------------
# Subcommand: cli-plan-tree
# ---------------------------------------------------------------------------
_DECISION_ULID_1 = "01KHDE6WWS2171PWW3GJEBXZ01"
_DECISION_ULID_2 = "01KHDE6WWS2171PWW3GJEBXZ02"
_DECISION_ULID_3 = "01KHDE6WWS2171PWW3GJEBXZ03"
_DECISION_ULID_4 = "01KHDE6WWS2171PWW3GJEBXZ04"
_DECISION_ULID_5 = "01KHDE6WWS2171PWW3GJEBXZ05"
def cli_plan_tree() -> None:
"""Verify ``agents plan tree`` CLI displays subplan hierarchy.
Creates real Decision objects forming a subplan tree:
- prompt_definition (root)
- strategy_choice
- subplan_parallel_spawn
- subplan_spawn (child A)
- subplan_spawn (child B)
Mocks the DI container and DecisionService, then invokes the
CLI command with ``--format json`` and verifies the JSON output
contains the expected decision types and hierarchy.
"""
# Build a realistic decision tree with subplan decisions
decisions = [
Decision(
decision_id=_DECISION_ULID_1,
plan_id=_ROOT_ULID,
parent_decision_id=None,
sequence_number=0,
decision_type=DecisionType.PROMPT_DEFINITION,
question="What is the plan prompt?",
chosen_option="Refactor code across monorepo modules",
confidence_score=1.0,
),
Decision(
decision_id=_DECISION_ULID_2,
plan_id=_ROOT_ULID,
parent_decision_id=_DECISION_ULID_1,
sequence_number=1,
decision_type=DecisionType.STRATEGY_CHOICE,
question="How to decompose the refactoring?",
chosen_option="Parallel subplans per module",
confidence_score=0.85,
),
Decision(
decision_id=_DECISION_ULID_3,
plan_id=_ROOT_ULID,
parent_decision_id=_DECISION_ULID_2,
sequence_number=2,
decision_type=DecisionType.SUBPLAN_PARALLEL_SPAWN,
question="Which subplans to run in parallel?",
chosen_option="Spawn API and UI subplans concurrently",
downstream_plan_ids=[_CHILD_A_ULID, _CHILD_B_ULID],
),
Decision(
decision_id=_DECISION_ULID_4,
plan_id=_ROOT_ULID,
parent_decision_id=_DECISION_ULID_3,
sequence_number=3,
decision_type=DecisionType.SUBPLAN_SPAWN,
question="Spawn subplan for API module?",
chosen_option="Spawn API refactor subplan",
downstream_plan_ids=[_CHILD_A_ULID],
),
Decision(
decision_id=_DECISION_ULID_5,
plan_id=_ROOT_ULID,
parent_decision_id=_DECISION_ULID_3,
sequence_number=4,
decision_type=DecisionType.SUBPLAN_SPAWN,
question="Spawn subplan for UI module?",
chosen_option="Spawn UI refactor subplan",
downstream_plan_ids=[_CHILD_B_ULID],
),
]
# Mock the DI container and DecisionService
mock_container = MagicMock()
mock_decision_svc = MagicMock()
mock_decision_svc.list_decisions.return_value = decisions
mock_container.resolve.return_value = mock_decision_svc
with patch(
"cleveragents.application.container.get_container",
return_value=mock_container,
):
result = runner.invoke(
plan_app,
["tree", _ROOT_ULID, "--format", "json"],
)
if result.exit_code != 0:
_fail(
f"plan tree rc={result.exit_code}\n"
f"output={result.output}\n"
f"exception={result.exception}"
)
# Verify the decision service was queried
mock_decision_svc.list_decisions.assert_called_once_with(_ROOT_ULID)
# Parse the JSON output and verify tree structure
try:
tree_data = json.loads(result.output)
except json.JSONDecodeError:
_fail(f"plan tree output is not valid JSON:\n{result.output}")
# Tree should contain decision nodes — look for subplan types
tree_str = json.dumps(tree_data)
if "subplan_parallel_spawn" not in tree_str:
_fail(
"tree output missing subplan_parallel_spawn decision\n"
f"output={result.output}"
)
if "subplan_spawn" not in tree_str:
_fail(f"tree output missing subplan_spawn decision\noutput={result.output}")
if "prompt_definition" not in tree_str:
_fail(f"tree output missing prompt_definition root\noutput={result.output}")
print("m4-cli-plan-tree-ok")
# ---------------------------------------------------------------------------
# Main dispatcher
# ---------------------------------------------------------------------------
@@ -711,6 +961,9 @@ _COMMANDS: dict[str, Callable[[], None]] = {
"merge-clean": merge_clean,
"merge-conflict": merge_conflict,
"parent-tracking": parent_tracking,
"cli-plan-use": cli_plan_use,
"cli-plan-execute": cli_plan_execute,
"cli-plan-tree": cli_plan_tree,
}
robot/m4_e2e_verification.robot
+31
View File
@@ -77,3 +77,34 @@ Parent Plan Tracks All Subplan Statuses
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m4-parent-tracking-ok
CLI Plan Use Creates Plan With Subplan Config
[Documentation] Invoke agents plan use local/refactor-action local/monorepo
... via the actual CLI (Typer CliRunner) with a mocked lifecycle
... service and verify the plan is created with subplan config.
${result}= Run Process ${PYTHON} ${HELPER} cli-plan-use cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m4-cli-plan-use-ok
CLI Plan Execute Transitions With Subplans
[Documentation] Invoke agents plan execute <plan_id> via the actual CLI
... (Typer CliRunner) with a mocked lifecycle service and
... verify the plan transitions to Execute phase with subplans.
${result}= Run Process ${PYTHON} ${HELPER} cli-plan-execute cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m4-cli-plan-execute-ok
CLI Plan Tree Displays Subplan Hierarchy
[Documentation] Invoke agents plan tree <plan_id> --format json via the
... actual CLI (Typer CliRunner) with mocked Decision objects
... forming a subplan tree. Verify JSON output contains
... subplan_spawn and subplan_parallel_spawn decision types.
${result}= Run Process ${PYTHON} ${HELPER} cli-plan-tree cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m4-cli-plan-tree-ok