fix(e2e): restore M5/WF14 tests broken by JSON envelope and stale config #9912

Merged
HAL9000 merged 2 commits from test/restore-e2e-tests into master 2026-05-01 01:18:01 +00:00
9 changed files with 54 additions and 0 deletions
+1
View File
@@ -37,6 +37,7 @@ Setup Test Environment
... that intentionally need real AI responses.
[Arguments] ${auto_apply_migrations}=${TRUE} ${mock_ai}=${TRUE}
Log Setting up test environment
Set Environment Variable NO_COLOR 1
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
${home}= Set Variable ${TEMPDIR}${/}.cleveragents_${safe_suite}
Run Keyword And Ignore Error Remove Directory ${home} recursive=True
+2
View File
@@ -22,6 +22,7 @@ E2E Suite Setup
... Does NOT enable mock AI — E2E tests use real providers.
... Propagates LLM API keys from the environment.
Log Setting up E2E test environment
Set Environment Variable NO_COLOR 1
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
${safe_suite}= Replace String ${safe_suite} . _
${safe_suite}= Replace String Using Regexp ${safe_suite} [^A-Za-z0-9_-] _
@@ -53,6 +54,7 @@ E2E Suite Teardown
Run Keyword And Ignore Error Remove Directory ${SUITE_HOME} recursive=True
Remove Environment Variable CLEVERAGENTS_HOME
Remove Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS
Remove Environment Variable NO_COLOR
Skip If No LLM Keys
[Documentation] Skip the current test if no LLM API keys are available.
+9
View File
@@ -48,6 +48,7 @@ from helper_e2e_common import ( # noqa: E402
setup_workspace,
write_yaml,
)
from helpers_common import reset_global_state # noqa: E402
from cleveragents.domain.models.core.action import Action, ActionState # noqa: E402
from cleveragents.domain.models.core.change import ( # noqa: E402
2
@@ -160,6 +161,7 @@ def _fail(msg: str) -> NoReturn:
def action_create_from_yaml() -> None:
"""Verify action create from YAML config via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_action_")
yaml_path = write_yaml(_VALID_ACTION_YAML)
try:
@@ -192,6 +194,7 @@ def action_create_from_yaml() -> None:
def resource_register_git_checkout() -> None:
"""Register a git-checkout resource via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_resource_")
repo_dir = init_bare_git_repo()
try:
@@ -236,6 +239,7 @@ def resource_register_git_checkout() -> None:
def project_create_and_link() -> None:
"""Create a project and link a resource via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_project_")
repo_dir = init_bare_git_repo()
try:
@@ -306,6 +310,7 @@ def plan_full_lifecycle() -> None:
end-to-end. ChangeSet verification uses the domain-level tests
(``changeset-invocations``), not this CLI lifecycle test.
"""
reset_global_state()
workspace = setup_workspace(prefix="m1_plan_")
repo_dir = init_bare_git_repo()
yaml_path = write_yaml(_VALID_ACTION_YAML)
@@ -416,6 +421,7 @@ def sqlite_persistence_check() -> None:
Uses the in-memory ChangeSetStore and domain model constructors
to verify data survives serialisation round-trips.
"""
reset_global_state()
# Create Action and verify serialisation
action = _mock_action()
action_dict = {
@@ -483,6 +489,7 @@ def changeset_from_tool_invocations() -> None:
Creates ToolInvocation records and verifies ChangeEntry objects
are linked via tool_name, not extracted from CLI output text.
"""
reset_global_state()
store, cid = _make_store()
# Simulate tool invocations producing ChangeEntry records
@@ -531,6 +538,7 @@ def sandbox_isolation_check() -> None:
Also verifies sandbox changes do not affect the original repo
until Apply (commit + merge).
"""
reset_global_state()
from cleveragents.infrastructure.sandbox.git_worktree import (
GitWorktreeSandbox,
)
@@ -592,6 +600,7 @@ def post_apply_commit_check() -> None:
Creates a git worktree sandbox, writes a file, commits, and
verifies the commit is visible in git log on the original branch.
"""
reset_global_state()
from cleveragents.infrastructure.sandbox.git_worktree import (
GitWorktreeSandbox,
)
+10
View File
@@ -46,6 +46,7 @@ from helper_e2e_common import ( # noqa: E402
setup_workspace,
write_yaml,
)
from helpers_common import reset_global_state # noqa: E402
from cleveragents.actor.compiler import ( # noqa: E402
ActorCompilationError,
@@ -113,6 +114,7 @@ def _validation_fail(inputs: dict[str, Any]) -> dict[str, Any]:
def actor_yaml_create_load() -> None:
"""Create a temporary actor YAML and load it via ActorLoader."""
reset_global_state()
tmp = Path(tempfile.mkdtemp(prefix="m2_actor_"))
yaml_content = (
"name: local/m2-test-actor\n"
@@ -162,6 +164,7 @@ def actor_yaml_create_load() -> None:
def actor_add_config_cli() -> None:
"""Add an actor via ``agents actor add --config`` using subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m2_actor_cli_")
config_dir = tempfile.mkdtemp(prefix="m2_actor_cfg_")
config_path = os.path.join(config_dir, "actor.json")
@@ -213,6 +216,7 @@ execution_actor: openai/gpt-4
def action_create() -> None:
"""Create an action via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m2_action_")
yaml_path = write_yaml(_ACTION_YAML)
try:
@@ -248,6 +252,7 @@ def action_create() -> None:
def plan_use_execute() -> None:
"""Run plan use and plan execute via real CLI subprocesses."""
reset_global_state()
workspace = setup_workspace(prefix="m2_plan_")
yaml_path = write_yaml(_ACTION_YAML)
try:
@@ -293,6 +298,7 @@ def plan_use_execute() -> None:
def actor_yaml_parse_validate() -> None:
"""Verify actor YAML files parse and validate correctly."""
reset_global_state()
# Test valid graph YAML
tmp = Path(tempfile.mkdtemp(prefix="m2_parse_"))
valid_yaml = (
@@ -356,6 +362,7 @@ def actor_yaml_parse_validate() -> None:
def actor_compile_stategraph() -> None:
"""Verify actors compile to LangGraph StateGraphs."""
reset_global_state()
project_root = Path(__file__).resolve().parents[1]
yaml_path = project_root / "examples" / "actors" / "simple_graph.yaml"
config = ActorConfigSchema.from_yaml_file(str(yaml_path))
@@ -398,6 +405,7 @@ def actor_compile_stategraph() -> None:
def tool_router_resolve_external() -> None:
"""Verify tool router can resolve external tools."""
reset_global_state()
registry = ToolRegistry()
ext_spec = ToolSpec(
name="external/custom-tool",
@@ -450,6 +458,7 @@ def tool_router_resolve_external() -> None:
def validation_runner_execute() -> None:
"""Verify validation runner executes required/informational validations."""
reset_global_state()
# Create a required validation
required_val = Validation.from_config(
{
@@ -522,6 +531,7 @@ def validation_runner_execute() -> None:
def changeset_multifile() -> None:
"""Verify multi-file generation produces correct ChangeSet."""
reset_global_state()
tmpdir = tempfile.mkdtemp(prefix="m2_changeset_")
capture = ChangeSetCapture(
plan_id="m2-plan-cs",
+5
View File
@@ -37,6 +37,7 @@ from helper_m4_e2e_common import ( # noqa: E402
_make_subplan_statuses,
_mock_parent_plan,
)
from helpers_common import reset_global_state # noqa: E402
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
2
@@ -68,6 +69,7 @@ _DECISION_SPAWN_UI = "01KHDE6WWS2171PWW3GJEBXZ05"
def plan_diff() -> None:
"""Verify merged results via ``agents plan diff`` (mocked service)."""
reset_global_state()
mock_apply_svc = MagicMock()
mock_apply_svc.diff.return_value = (
"--- a/src/api.py\n"
@@ -145,6 +147,7 @@ def cli_plan_use() -> None:
Typer's CliRunner. Asserts the service receives the correct action
name and project, and the CLI exits cleanly.
"""
reset_global_state()
mock_service = MagicMock()
# Mock action lookup -- use_action needs get_action_by_name first
@@ -221,6 +224,7 @@ def cli_plan_execute() -> None:
Asserts the plan transitions to Execute phase and the CLI renders
the plan details.
"""
reset_global_state()
mock_service = MagicMock()
# get_plan is called for the read-only guard
@@ -306,6 +310,7 @@ def cli_plan_tree() -> None:
``tree`` command resolves the ``DecisionService`` directly from
the DI container (not via the lifecycle service helper).
"""
reset_global_state()
# Build a realistic decision tree with subplan decisions
decisions = [
Decision(
+12
View File
@@ -6,6 +6,8 @@ import json
from pathlib import Path
from typing import Any
from helpers_common import reset_global_state
from cleveragents.domain.models.core.context_policy import (
ContextView,
ProjectContextPolicy,
@@ -21,24 +23,28 @@ _FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" /
def load_acms_context_policy_fixture() -> dict[str, Any]:
"""Load the ACMS context policy fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "acms_context_policy.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def load_large_project_context_fixture() -> dict[str, Any]:
"""Load the large project context fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "large_project_context.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def load_context_analysis_fixture() -> dict[str, Any]:
"""Load the context analysis results fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "context_analysis_results.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def resolve_view_for_phase(phase: str) -> dict[str, Any]:
"""Resolve a view from an empty policy for the given phase."""
reset_global_state()
policy = ProjectContextPolicy()
view = policy.resolve_view(phase)
return {
@@ -53,6 +59,7 @@ def resolve_view_for_phase(phase: str) -> dict[str, Any]:
def resolve_strategize_inheriting_default() -> dict[str, Any]:
"""Resolve strategize view that inherits from default."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_paths=["src/**/*.py"],
@@ -69,6 +76,7 @@ def resolve_strategize_inheriting_default() -> dict[str, Any]:
def resolve_strategize_with_override() -> dict[str, Any]:
"""Resolve strategize view with explicit override."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_paths=["src/**/*.py"],
@@ -92,6 +100,7 @@ def check_budget_enforcement(
within_file: int,
) -> dict[str, bool]:
"""Check if files exceed or fit within file size budget."""
reset_global_state()
view = ContextView(max_file_size=int(max_size))
limit = view.max_file_size
assert limit is not None
@@ -107,6 +116,7 @@ def check_total_budget_enforcement(
within_total: int,
) -> dict[str, bool]:
"""Check if aggregate context exceeds total size budget."""
reset_global_state()
view = ContextView(max_total_size=int(max_total))
limit = view.max_total_size
assert limit is not None
@@ -118,6 +128,7 @@ def check_total_budget_enforcement(
def attempt_resolve_invalid_phase(phase: str) -> dict[str, str]:
"""Try resolving an invalid phase and capture the error."""
reset_global_state()
policy = ProjectContextPolicy()
try:
policy.resolve_view(phase)
@@ -131,6 +142,7 @@ def create_multi_project_context(
count_b: int,
) -> dict[str, int]:
"""Simulate independent context for two projects."""
reset_global_state()
proj_a = [{"path": f"src/a_{i}.py", "size": 1024} for i in range(int(count_a))]
proj_b = [{"path": f"src/b_{i}.py", "size": 1024} for i in range(int(count_b))]
return {
+8
View File
@@ -7,6 +7,7 @@ from typing import Any, cast
from unittest.mock import MagicMock
from helper_m5_e2e_support import _create_project, _fail, _setup_db
from helpers_common import reset_global_state
from cleveragents.application.services.context_phase_analysis import (
analyze_phase_summaries,
@@ -30,6 +31,7 @@ from cleveragents.domain.models.core.project import ContextConfig, NamespacedPro
def context_tier_config() -> None:
"""Verify ContextConfig with hot/warm/cold tier management."""
reset_global_state()
default_cfg = ContextConfig()
if default_cfg.hot_max_tokens is not None:
_fail(f"default hot_max_tokens should be None: {default_cfg.hot_max_tokens}")
@@ -79,6 +81,7 @@ def context_tier_config() -> None:
def context_policy_set_show() -> None:
"""Set a context policy via SQL helpers and read it back."""
reset_global_state()
proj_repo, _link_repo, _svc, sf = _setup_db()
_create_project(proj_repo, "local/policy-test")
@@ -129,6 +132,7 @@ def context_policy_set_show() -> None:
def acms_phase_inheritance() -> None:
"""Verify ACMS view inheritance: default -> strategize -> execute -> apply."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_resources=["*"],
@@ -179,6 +183,7 @@ def acms_phase_inheritance() -> None:
def acms_scoped_context() -> None:
"""Verify ACMS produces scoped context output per phase."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_resources=["*"],
@@ -323,6 +328,7 @@ def acms_scoped_context() -> None:
def context_policy_clear() -> None:
"""Verify clearing a phase view causes inheritance fallback."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(include_resources=["all-*"]),
strategize_view=ContextView(include_resources=["strat-*"]),
@@ -373,6 +379,7 @@ def context_policy_clear() -> None:
def context_view_validation() -> None:
"""Verify ContextView validation rules."""
reset_global_state()
cv = ContextView(max_file_size=1, max_total_size=1)
if cv.max_file_size != 1:
_fail(f"max_file_size should be 1: {cv.max_file_size}")
@@ -409,6 +416,7 @@ def context_view_validation() -> None:
def llm_execute_acms_context() -> None:
"""Verify execute-phase LLM prompt uses ACMS assembled execute context."""
reset_global_state()
proj_repo, _link_repo, _svc, sf = _setup_db()
_create_project(proj_repo, "local/exec-ctx")
+3
View File
@@ -6,6 +6,7 @@ import sys
from pathlib import Path
from typing import Any, NoReturn, cast
from helpers_common import reset_global_state
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
@@ -49,6 +50,7 @@ def _setup_db() -> tuple[
Any,
]:
"""Create an in-memory SQLite DB with all tables."""
reset_global_state()
engine = create_engine("sqlite:///:memory:", echo=False)
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine, expire_on_commit=False)()
@@ -70,6 +72,7 @@ def _create_project(
description: str | None = None,
) -> NamespacedProject:
"""Create and return a NamespacedProject via the repository."""
reset_global_state()
parsed = parse_namespaced_name(name)
proj = NamespacedProject(
name=parsed.name,
+4
View File
@@ -45,6 +45,7 @@ from helper_m5_e2e_support import ( # noqa: E402
_fail,
_setup_db,
)
from helpers_common import reset_global_state # noqa: E402
from cleveragents.application.services.repo_indexing_service import ( # noqa: E402
RepoIndexingService,
@@ -62,6 +63,7 @@ def project_create_large() -> None:
Verifies project creation, resource registration, linking,
and that the project show reflects the linked resource.
"""
reset_global_state()
proj_repo, link_repo, svc, _sf = _setup_db()
proj = _create_project(
@@ -114,6 +116,7 @@ def project_create_large() -> None:
def resource_register_link() -> None:
"""Register a git-checkout resource and link it to a project."""
reset_global_state()
proj_repo, link_repo, svc, _sf = _setup_db()
proj = _create_project(proj_repo, "local/link-test")
@@ -162,6 +165,7 @@ def resource_register_link() -> None:
def indexing_complete() -> None:
"""Verify indexing completes: resource is linked, project shows OK."""
reset_global_state()
proj_repo, link_repo, svc, sf = _setup_db()
proj = _create_project(proj_repo, "local/idx-project")