317d015260
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / build (pull_request) Successful in 17s
CI / quality (pull_request) Successful in 18s
CI / e2e_tests (pull_request) Successful in 27s
CI / security (pull_request) Successful in 38s
CI / typecheck (pull_request) Successful in 39s
CI / unit_tests (pull_request) Successful in 3m0s
CI / docker (pull_request) Successful in 10s
CI / integration_tests (pull_request) Successful in 3m29s
CI / coverage (pull_request) Successful in 5m54s
CI / lint (push) Successful in 13s
CI / build (push) Successful in 14s
CI / quality (push) Successful in 17s
CI / security (push) Successful in 38s
CI / e2e_tests (push) Successful in 36s
CI / typecheck (push) Successful in 39s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m59s
CI / integration_tests (push) Successful in 3m24s
CI / docker (push) Successful in 38s
CI / coverage (push) Successful in 5m54s
CI / benchmark-publish (push) Successful in 19m34s
CI / benchmark-regression (pull_request) Successful in 36m21s
Strengthen M4 E2E CLI acceptance tests and address all review findings from the self-review on PR #681 (3 blocking, 7 non-blocking issues). Blocking fixes: - Remove tautological domain assertions from cli_plan_execute() that verified values the test itself constructed via _mock_parent_plan and could never fail. Replace with a comment documenting that subplan info cannot be verified from CLI output alone. - Fix unguarded positional arg access in plan_diff() that would raise IndexError if the production CLI passes plan_id as a keyword argument. Now uses kwargs-with-positional-fallback pattern. - Split the 1074-line helper_m4_e2e_verification.py into four focused modules under the CONTRIBUTING.md 500-line limit: helper_m4_e2e_common.py (237 lines) — constants, helpers, factories helper_m4_e2e_domain.py (495 lines) — domain model tests helper_m4_e2e_cli.py (426 lines) — CLI integration tests helper_m4_e2e_verification.py (81 lines) — thin dispatcher Non-blocking improvements: - Add ProjectLink isinstance type guard before iterating project_links. - Wrap all assert_called_once*() calls in try/except with _fail() pattern to produce standardised FAIL: diagnostics instead of raw AssertionError. - Make project_links extraction symmetric with action_name (kwargs + positional fallback at index 1). - Replace broad "execute" substring match with word-boundary regex r"\bexecute\b" to avoid false positives from incidental substrings. - Extract _make_subplan_status() factory, _assert_exit_code() and _assert_mock_called_once*() wrappers to eliminate DRY violations (SubplanStatus construction repeated 3x, exit-code checks 4x). - Replace non-deterministic datetime.now() calls with frozen constant _FROZEN_NOW = datetime(2026, 3, 1, 12, 0, 0) for deterministic fixtures. - Rename generic _DECISION_ULID_N constants to descriptive names (_DECISION_PROMPT_DEF, _DECISION_STRATEGY, _DECISION_PARALLEL_SPAWN, _DECISION_SPAWN_API, _DECISION_SPAWN_UI). Quality gates: - lint: PASS - format: PASS (1404 files unchanged) - typecheck: PASS (0 errors) - unit_tests: 10,431 scenarios, 0 failures - integration_tests: 1,452 tests, 0 failures - coverage_report: PASS (98%, threshold 97%) - security_scan: PASS - dead_code: PASS - docs: PASS - build: PASS ISSUES CLOSED: #495
245 lines
8.0 KiB
Python
245 lines
8.0 KiB
Python
"""Shared constants, utilities, and mock factories for M4 E2E verification tests.
|
|
|
|
Provides reusable test infrastructure for both domain-model tests
|
|
(``helper_m4_e2e_domain``), merge tests (``helper_m4_e2e_merge``),
|
|
and CLI integration tests (``helper_m4_e2e_cli``, ``helper_m4_e2e_cli_errors``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, NoReturn
|
|
from unittest.mock import MagicMock
|
|
|
|
# Ensure the src directory is on the import path. Uses ``append`` rather
|
|
# than ``insert(0, ...)`` to avoid shadowing stdlib modules if a file
|
|
# in ``src/`` were ever named identically to a stdlib module.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.append(_SRC)
|
|
|
|
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
|
ExecutionMode,
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
SubplanAttempt,
|
|
SubplanConfig,
|
|
SubplanMergeStrategy,
|
|
SubplanStatus,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
# Typer's CliRunner wraps Click's Result; importing via click.testing
|
|
# is the canonical path since typer.testing re-exports CliRunner but
|
|
# does not re-export Result.
|
|
from click.testing import Result as CliResult
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_ROOT_ULID = "01KHDE6WWS2171PWW3GJEBXZ8R"
|
|
_CHILD_A_ULID = "01KHDE6WWS2171PWW3GJEBXZ8A"
|
|
_CHILD_B_ULID = "01KHDE6WWS2171PWW3GJEBXZ8B"
|
|
_CHILD_C_ULID = "01KHDE6WWS2171PWW3GJEBXZ8C"
|
|
|
|
# Frozen timestamp for deterministic fixtures. Avoids non-deterministic
|
|
# ``datetime.now()`` calls that produce subtly inconsistent timestamps
|
|
# across fixture objects within a single test.
|
|
_FROZEN_NOW = datetime(2026, 3, 1, 12, 0, 0)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared test helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _fail(msg: str) -> NoReturn:
|
|
"""Print failure message to stderr and exit 1."""
|
|
print(f"FAIL: {msg}", file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def _assert_mock_called_once(mock: MagicMock, label: str) -> None:
|
|
"""Wrap ``assert_called_once`` with standardised ``_fail()`` diagnostics.
|
|
|
|
Raw ``AssertionError`` from ``unittest.mock`` bypasses the ``FAIL:``
|
|
prefix pattern used by Robot Framework test evaluation; this wrapper
|
|
converts it.
|
|
"""
|
|
try:
|
|
mock.assert_called_once()
|
|
except AssertionError as exc:
|
|
_fail(f"{label}: {exc}")
|
|
|
|
|
|
def _assert_mock_called_once_with(
|
|
mock: MagicMock,
|
|
label: str,
|
|
*args: object,
|
|
**kwargs: object,
|
|
) -> None:
|
|
"""Wrap ``assert_called_once_with`` with ``_fail()`` diagnostics."""
|
|
try:
|
|
mock.assert_called_once_with(*args, **kwargs)
|
|
except AssertionError as exc:
|
|
_fail(f"{label}: {exc}")
|
|
|
|
|
|
def _assert_exit_code(result: CliResult, label: str) -> None:
|
|
"""Verify CLI exit code is 0, failing with full diagnostic output."""
|
|
if result.exit_code != 0:
|
|
_fail(
|
|
f"{label} rc={result.exit_code}\n"
|
|
f"output={result.output}\n"
|
|
f"exception={result.exception}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SubplanStatus factory (DRY extraction)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_subplan_status(
|
|
subplan_id: str,
|
|
*,
|
|
action_name: str = "local/m4-verify-action",
|
|
target_resources: list[str] | None = None,
|
|
status: ProcessingState = ProcessingState.QUEUED,
|
|
started_at: datetime | None = None,
|
|
completed_at: datetime | None = None,
|
|
changeset_summary: str | None = None,
|
|
files_changed: int = 0,
|
|
error: str | None = None,
|
|
attempt_number: int = 1,
|
|
previous_attempts: list[SubplanAttempt] | None = None,
|
|
) -> SubplanStatus:
|
|
"""Create a ``SubplanStatus`` with sensible test defaults."""
|
|
return SubplanStatus(
|
|
subplan_id=subplan_id,
|
|
action_name=action_name,
|
|
target_resources=target_resources or [],
|
|
status=status,
|
|
started_at=started_at,
|
|
completed_at=completed_at,
|
|
changeset_summary=changeset_summary,
|
|
files_changed=files_changed,
|
|
error=error,
|
|
attempt_number=attempt_number,
|
|
previous_attempts=previous_attempts or [],
|
|
)
|
|
|
|
|
|
def _make_subplan_statuses() -> list[SubplanStatus]:
|
|
"""Create three SubplanStatus entries (A=COMPLETE, B=COMPLETE, C=QUEUED).
|
|
|
|
This is the default three-child status set used by many M4 tests.
|
|
"""
|
|
return [
|
|
_make_subplan_status(
|
|
_CHILD_A_ULID,
|
|
target_resources=["res-api"],
|
|
status=ProcessingState.COMPLETE,
|
|
started_at=_FROZEN_NOW,
|
|
completed_at=_FROZEN_NOW,
|
|
changeset_summary="Updated API endpoints",
|
|
files_changed=3,
|
|
),
|
|
_make_subplan_status(
|
|
_CHILD_B_ULID,
|
|
target_resources=["res-ui"],
|
|
status=ProcessingState.COMPLETE,
|
|
started_at=_FROZEN_NOW,
|
|
completed_at=_FROZEN_NOW,
|
|
changeset_summary="Updated UI components",
|
|
files_changed=5,
|
|
),
|
|
_make_subplan_status(
|
|
_CHILD_C_ULID,
|
|
target_resources=["res-db"],
|
|
),
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plan mock factories
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _mock_parent_plan(
|
|
phase: PlanPhase = PlanPhase.EXECUTE,
|
|
state: ProcessingState = ProcessingState.PROCESSING,
|
|
subplan_statuses: list[SubplanStatus] | None = None,
|
|
subplan_config: SubplanConfig | None = None,
|
|
*,
|
|
read_only: bool = False,
|
|
) -> Plan:
|
|
"""Create a parent plan with subplan config."""
|
|
return Plan(
|
|
identity=PlanIdentity(plan_id=_ROOT_ULID),
|
|
namespaced_name=NamespacedName.parse("local/m4-parent-plan"),
|
|
description="M4 verification parent plan with subplans",
|
|
definition_of_done="All subplans complete successfully",
|
|
action_name="local/m4-verify-action",
|
|
phase=phase,
|
|
processing_state=state,
|
|
project_links=[ProjectLink(project_name="local/m4-project")],
|
|
arguments={},
|
|
arguments_order=[],
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
timestamps=PlanTimestamps(created_at=_FROZEN_NOW, updated_at=_FROZEN_NOW),
|
|
created_by=None,
|
|
reusable=True,
|
|
read_only=read_only,
|
|
subplan_config=subplan_config
|
|
or SubplanConfig(
|
|
execution_mode=ExecutionMode.PARALLEL,
|
|
merge_strategy=SubplanMergeStrategy.GIT_THREE_WAY,
|
|
max_parallel=3,
|
|
fail_fast=False,
|
|
retry_failed=True,
|
|
max_retries=2,
|
|
),
|
|
subplan_statuses=subplan_statuses or [],
|
|
)
|
|
|
|
|
|
def _mock_child_plan(
|
|
child_id: str,
|
|
parent_id: str = _ROOT_ULID,
|
|
root_id: str = _ROOT_ULID,
|
|
phase: PlanPhase = PlanPhase.EXECUTE,
|
|
state: ProcessingState = ProcessingState.QUEUED,
|
|
) -> Plan:
|
|
"""Create a child subplan."""
|
|
return Plan(
|
|
identity=PlanIdentity(
|
|
plan_id=child_id,
|
|
parent_plan_id=parent_id,
|
|
root_plan_id=root_id,
|
|
),
|
|
namespaced_name=NamespacedName.parse(f"local/m4-child-{child_id[-1].lower()}"),
|
|
description=f"M4 child subplan {child_id[-1]}",
|
|
definition_of_done="Subplan criteria pass",
|
|
action_name="local/m4-verify-action",
|
|
phase=phase,
|
|
processing_state=state,
|
|
project_links=[ProjectLink(project_name="local/m4-project")],
|
|
arguments={},
|
|
arguments_order=[],
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
timestamps=PlanTimestamps(created_at=_FROZEN_NOW, updated_at=_FROZEN_NOW),
|
|
created_by=None,
|
|
reusable=True,
|
|
read_only=False,
|
|
)
|