Compare commits

...

1 Commits

Author SHA1 Message Date
HAL9000 a27687baa0 fix(e2e): restore M5/WF14 tests broken by JSON envelope and stale config
CI / push-validation (pull_request) Successful in 24s
CI / helm (pull_request) Successful in 34s
CI / build (pull_request) Successful in 53s
CI / lint (pull_request) Failing after 1m0s
CI / quality (pull_request) Successful in 1m12s
CI / typecheck (pull_request) Successful in 1m42s
CI / security (pull_request) Successful in 1m42s
CI / benchmark-publish (pull_request) Has been skipped
CI / e2e_tests (pull_request) Successful in 3m49s
CI / integration_tests (pull_request) Failing after 3m57s
CI / unit_tests (pull_request) Failing after 6m17s
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / status-check (pull_request) Failing after 3s
- Add NO_COLOR=1 to E2E Suite Setup and Test Environment to prevent
  Rich from injecting ANSI escape codes into JSON output, which breaks
  Extract JSON From Stdout and JSON assertions in M5/WF14 acceptance tests.

- Add reset_global_state() calls to all M1/M2/M4/M5 E2E verification
  helpers to clear Settings singleton, DI container, provider registry,
  and SQLAlchemy engine cache between parallel pabot worker runs.

- Propagate NO_COLOR=1 at suite-setup level in E2E and integration
  test resource files so all Run Process subprocesses inherit clean
  terminal output mode.

Root causes:
1. Missing NO_COLOR=1 caused Rich ANSI codes in CLI JSON output,
   making json.loads() failures in robot Extract JSON From Stdout
2. Stale Settings singleton carried provider keys and DB paths between
   pabot workers, causing UNIQUE constraint violations and stale config
   lookups in context policy and server mode E2E tests.

Related: PR #9912
2026-04-29 01:38:57 +00:00
9 changed files with 58 additions and 0 deletions
+1
View File
@@ -37,6 +37,7 @@ Setup Test Environment
... that intentionally need real AI responses.
[Arguments] ${auto_apply_migrations}=${TRUE} ${mock_ai}=${TRUE}
Log Setting up test environment
Set Environment Variable NO_COLOR 1
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
${home}= Set Variable ${TEMPDIR}${/}.cleveragents_${safe_suite}
Run Keyword And Ignore Error Remove Directory ${home} recursive=True
+2
View File
@@ -22,6 +22,7 @@ E2E Suite Setup
... Does NOT enable mock AI — E2E tests use real providers.
... Propagates LLM API keys from the environment.
Log Setting up E2E test environment
Set Environment Variable NO_COLOR 1
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
${safe_suite}= Replace String ${safe_suite} . _
${safe_suite}= Replace String Using Regexp ${safe_suite} [^A-Za-z0-9_-] _
@@ -53,6 +54,7 @@ E2E Suite Teardown
Run Keyword And Ignore Error Remove Directory ${SUITE_HOME} recursive=True
Remove Environment Variable CLEVERAGENTS_HOME
Remove Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS
Remove Environment Variable NO_COLOR
Skip If No LLM Keys
[Documentation] Skip the current test if no LLM API keys are available.
+9
View File
@@ -57,6 +57,7 @@ from cleveragents.domain.models.core.change import ( # noqa: E402
ToolInvocation,
)
from cleveragents.domain.models.core.plan import ( # noqa: E402
from helpers_common import reset_global_state # noqa: E402
AutomationProfileProvenance,
AutomationProfileRef,
NamespacedName,
@@ -160,6 +161,7 @@ def _fail(msg: str) -> NoReturn:
def action_create_from_yaml() -> None:
"""Verify action create from YAML config via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_action_")
yaml_path = write_yaml(_VALID_ACTION_YAML)
try:
@@ -192,6 +194,7 @@ def action_create_from_yaml() -> None:
def resource_register_git_checkout() -> None:
"""Register a git-checkout resource via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_resource_")
repo_dir = init_bare_git_repo()
try:
@@ -236,6 +239,7 @@ def resource_register_git_checkout() -> None:
def project_create_and_link() -> None:
"""Create a project and link a resource via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m1_project_")
repo_dir = init_bare_git_repo()
try:
@@ -306,6 +310,7 @@ def plan_full_lifecycle() -> None:
end-to-end. ChangeSet verification uses the domain-level tests
(``changeset-invocations``), not this CLI lifecycle test.
"""
reset_global_state()
workspace = setup_workspace(prefix="m1_plan_")
repo_dir = init_bare_git_repo()
yaml_path = write_yaml(_VALID_ACTION_YAML)
@@ -416,6 +421,7 @@ def sqlite_persistence_check() -> None:
Uses the in-memory ChangeSetStore and domain model constructors
to verify data survives serialisation round-trips.
"""
reset_global_state()
# Create Action and verify serialisation
action = _mock_action()
action_dict = {
@@ -483,6 +489,7 @@ def changeset_from_tool_invocations() -> None:
Creates ToolInvocation records and verifies ChangeEntry objects
are linked via tool_name, not extracted from CLI output text.
"""
reset_global_state()
store, cid = _make_store()
# Simulate tool invocations producing ChangeEntry records
@@ -531,6 +538,7 @@ def sandbox_isolation_check() -> None:
Also verifies sandbox changes do not affect the original repo
until Apply (commit + merge).
"""
reset_global_state()
from cleveragents.infrastructure.sandbox.git_worktree import (
GitWorktreeSandbox,
)
@@ -592,6 +600,7 @@ def post_apply_commit_check() -> None:
Creates a git worktree sandbox, writes a file, commits, and
verifies the commit is visible in git log on the original branch.
"""
reset_global_state()
from cleveragents.infrastructure.sandbox.git_worktree import (
GitWorktreeSandbox,
)
+11
View File
@@ -76,6 +76,8 @@ from cleveragents.tool.router import ( # noqa: E402
from cleveragents.tool.runner import ToolRunner # noqa: E402
from cleveragents.tool.runtime import ToolSpec # noqa: E402
from helpers_common import reset_global_state # noqa: E402
def _fail(msg: str) -> NoReturn:
"""Print failure message and exit."""
@@ -113,6 +115,7 @@ def _validation_fail(inputs: dict[str, Any]) -> dict[str, Any]:
def actor_yaml_create_load() -> None:
"""Create a temporary actor YAML and load it via ActorLoader."""
reset_global_state()
tmp = Path(tempfile.mkdtemp(prefix="m2_actor_"))
yaml_content = (
"name: local/m2-test-actor\n"
@@ -162,6 +165,7 @@ def actor_yaml_create_load() -> None:
def actor_add_config_cli() -> None:
"""Add an actor via ``agents actor add --config`` using subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m2_actor_cli_")
config_dir = tempfile.mkdtemp(prefix="m2_actor_cfg_")
config_path = os.path.join(config_dir, "actor.json")
@@ -213,6 +217,7 @@ execution_actor: openai/gpt-4
def action_create() -> None:
"""Create an action via real CLI subprocess."""
reset_global_state()
workspace = setup_workspace(prefix="m2_action_")
yaml_path = write_yaml(_ACTION_YAML)
try:
@@ -248,6 +253,7 @@ def action_create() -> None:
def plan_use_execute() -> None:
"""Run plan use and plan execute via real CLI subprocesses."""
reset_global_state()
workspace = setup_workspace(prefix="m2_plan_")
yaml_path = write_yaml(_ACTION_YAML)
try:
@@ -293,6 +299,7 @@ def plan_use_execute() -> None:
def actor_yaml_parse_validate() -> None:
"""Verify actor YAML files parse and validate correctly."""
reset_global_state()
# Test valid graph YAML
tmp = Path(tempfile.mkdtemp(prefix="m2_parse_"))
valid_yaml = (
@@ -356,6 +363,7 @@ def actor_yaml_parse_validate() -> None:
def actor_compile_stategraph() -> None:
"""Verify actors compile to LangGraph StateGraphs."""
reset_global_state()
project_root = Path(__file__).resolve().parents[1]
yaml_path = project_root / "examples" / "actors" / "simple_graph.yaml"
config = ActorConfigSchema.from_yaml_file(str(yaml_path))
@@ -398,6 +406,7 @@ def actor_compile_stategraph() -> None:
def tool_router_resolve_external() -> None:
"""Verify tool router can resolve external tools."""
reset_global_state()
registry = ToolRegistry()
ext_spec = ToolSpec(
name="external/custom-tool",
@@ -450,6 +459,7 @@ def tool_router_resolve_external() -> None:
def validation_runner_execute() -> None:
"""Verify validation runner executes required/informational validations."""
reset_global_state()
# Create a required validation
required_val = Validation.from_config(
{
@@ -522,6 +532,7 @@ def validation_runner_execute() -> None:
def changeset_multifile() -> None:
"""Verify multi-file generation produces correct ChangeSet."""
reset_global_state()
tmpdir = tempfile.mkdtemp(prefix="m2_changeset_")
capture = ChangeSetCapture(
plan_id="m2-plan-cs",
+5
View File
@@ -45,6 +45,7 @@ from cleveragents.domain.models.core.decision import ( # noqa: E402
DecisionType,
)
from cleveragents.domain.models.core.plan import ( # noqa: E402
from helpers_common import reset_global_state # noqa: E402
NamespacedName,
PlanPhase,
ProcessingState,
@@ -68,6 +69,7 @@ _DECISION_SPAWN_UI = "01KHDE6WWS2171PWW3GJEBXZ05"
def plan_diff() -> None:
"""Verify merged results via ``agents plan diff`` (mocked service)."""
reset_global_state()
mock_apply_svc = MagicMock()
mock_apply_svc.diff.return_value = (
"--- a/src/api.py\n"
@@ -145,6 +147,7 @@ def cli_plan_use() -> None:
Typer's CliRunner. Asserts the service receives the correct action
name and project, and the CLI exits cleanly.
"""
reset_global_state()
mock_service = MagicMock()
# Mock action lookup -- use_action needs get_action_by_name first
@@ -221,6 +224,7 @@ def cli_plan_execute() -> None:
Asserts the plan transitions to Execute phase and the CLI renders
the plan details.
"""
reset_global_state()
mock_service = MagicMock()
# get_plan is called for the read-only guard
@@ -306,6 +310,7 @@ def cli_plan_tree() -> None:
``tree`` command resolves the ``DecisionService`` directly from
the DI container (not via the lifecycle service helper).
"""
reset_global_state()
# Build a realistic decision tree with subplan decisions
decisions = [
Decision(
+12
View File
@@ -6,6 +6,8 @@ import json
from pathlib import Path
from typing import Any
from helpers_common import reset_global_state
from cleveragents.domain.models.core.context_policy import (
ContextView,
ProjectContextPolicy,
@@ -21,24 +23,28 @@ _FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" /
def load_acms_context_policy_fixture() -> dict[str, Any]:
"""Load the ACMS context policy fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "acms_context_policy.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def load_large_project_context_fixture() -> dict[str, Any]:
"""Load the large project context fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "large_project_context.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def load_context_analysis_fixture() -> dict[str, Any]:
"""Load the context analysis results fixture file."""
reset_global_state()
fixture_path = _FIXTURES_DIR / "context_analysis_results.json"
return json.loads(fixture_path.read_text(encoding="utf-8"))
def resolve_view_for_phase(phase: str) -> dict[str, Any]:
"""Resolve a view from an empty policy for the given phase."""
reset_global_state()
policy = ProjectContextPolicy()
view = policy.resolve_view(phase)
return {
@@ -53,6 +59,7 @@ def resolve_view_for_phase(phase: str) -> dict[str, Any]:
def resolve_strategize_inheriting_default() -> dict[str, Any]:
"""Resolve strategize view that inherits from default."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_paths=["src/**/*.py"],
@@ -69,6 +76,7 @@ def resolve_strategize_inheriting_default() -> dict[str, Any]:
def resolve_strategize_with_override() -> dict[str, Any]:
"""Resolve strategize view with explicit override."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_paths=["src/**/*.py"],
@@ -92,6 +100,7 @@ def check_budget_enforcement(
within_file: int,
) -> dict[str, bool]:
"""Check if files exceed or fit within file size budget."""
reset_global_state()
view = ContextView(max_file_size=int(max_size))
limit = view.max_file_size
assert limit is not None
@@ -107,6 +116,7 @@ def check_total_budget_enforcement(
within_total: int,
) -> dict[str, bool]:
"""Check if aggregate context exceeds total size budget."""
reset_global_state()
view = ContextView(max_total_size=int(max_total))
limit = view.max_total_size
assert limit is not None
@@ -118,6 +128,7 @@ def check_total_budget_enforcement(
def attempt_resolve_invalid_phase(phase: str) -> dict[str, str]:
"""Try resolving an invalid phase and capture the error."""
reset_global_state()
policy = ProjectContextPolicy()
try:
policy.resolve_view(phase)
@@ -131,6 +142,7 @@ def create_multi_project_context(
count_b: int,
) -> dict[str, int]:
"""Simulate independent context for two projects."""
reset_global_state()
proj_a = [{"path": f"src/a_{i}.py", "size": 1024} for i in range(int(count_a))]
proj_b = [{"path": f"src/b_{i}.py", "size": 1024} for i in range(int(count_b))]
return {
+9
View File
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock
from helper_m5_e2e_support import _create_project, _fail, _setup_db
from helpers_common import reset_global_state
from cleveragents.application.services.context_phase_analysis import (
analyze_phase_summaries,
)
@@ -30,6 +32,7 @@ from cleveragents.domain.models.core.project import ContextConfig, NamespacedPro
def context_tier_config() -> None:
"""Verify ContextConfig with hot/warm/cold tier management."""
reset_global_state()
default_cfg = ContextConfig()
if default_cfg.hot_max_tokens is not None:
_fail(f"default hot_max_tokens should be None: {default_cfg.hot_max_tokens}")
@@ -79,6 +82,7 @@ def context_tier_config() -> None:
def context_policy_set_show() -> None:
"""Set a context policy via SQL helpers and read it back."""
reset_global_state()
proj_repo, _link_repo, _svc, sf = _setup_db()
_create_project(proj_repo, "local/policy-test")
@@ -129,6 +133,7 @@ def context_policy_set_show() -> None:
def acms_phase_inheritance() -> None:
"""Verify ACMS view inheritance: default -> strategize -> execute -> apply."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_resources=["*"],
@@ -179,6 +184,7 @@ def acms_phase_inheritance() -> None:
def acms_scoped_context() -> None:
"""Verify ACMS produces scoped context output per phase."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(
include_resources=["*"],
@@ -323,6 +329,7 @@ def acms_scoped_context() -> None:
def context_policy_clear() -> None:
"""Verify clearing a phase view causes inheritance fallback."""
reset_global_state()
policy = ProjectContextPolicy(
default_view=ContextView(include_resources=["all-*"]),
strategize_view=ContextView(include_resources=["strat-*"]),
@@ -373,6 +380,7 @@ def context_policy_clear() -> None:
def context_view_validation() -> None:
"""Verify ContextView validation rules."""
reset_global_state()
cv = ContextView(max_file_size=1, max_total_size=1)
if cv.max_file_size != 1:
_fail(f"max_file_size should be 1: {cv.max_file_size}")
@@ -409,6 +417,7 @@ def context_view_validation() -> None:
def llm_execute_acms_context() -> None:
"""Verify execute-phase LLM prompt uses ACMS assembled execute context."""
reset_global_state()
proj_repo, _link_repo, _svc, sf = _setup_db()
_create_project(proj_repo, "local/exec-ctx")
+4
View File
@@ -6,6 +6,8 @@ import sys
from pathlib import Path
from typing import Any, NoReturn, cast
from helpers_common import reset_global_state
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
@@ -49,6 +51,7 @@ def _setup_db() -> tuple[
Any,
]:
"""Create an in-memory SQLite DB with all tables."""
reset_global_state()
engine = create_engine("sqlite:///:memory:", echo=False)
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine, expire_on_commit=False)()
@@ -70,6 +73,7 @@ def _create_project(
description: str | None = None,
) -> NamespacedProject:
"""Create and return a NamespacedProject via the repository."""
reset_global_state()
parsed = parse_namespaced_name(name)
proj = NamespacedProject(
name=parsed.name,
+5
View File
@@ -51,6 +51,8 @@ from cleveragents.application.services.repo_indexing_service import ( # noqa: E
)
from cleveragents.domain.models.core.repo_index import IndexStatus # noqa: E402
from helpers_common import reset_global_state # noqa: E402
# -------------------------------------------------------------------
# Subcommand: project-create-large
# -------------------------------------------------------------------
@@ -62,6 +64,7 @@ def project_create_large() -> None:
Verifies project creation, resource registration, linking,
and that the project show reflects the linked resource.
"""
reset_global_state()
proj_repo, link_repo, svc, _sf = _setup_db()
proj = _create_project(
@@ -114,6 +117,7 @@ def project_create_large() -> None:
def resource_register_link() -> None:
"""Register a git-checkout resource and link it to a project."""
reset_global_state()
proj_repo, link_repo, svc, _sf = _setup_db()
proj = _create_project(proj_repo, "local/link-test")
@@ -162,6 +166,7 @@ def resource_register_link() -> None:
def indexing_complete() -> None:
"""Verify indexing completes: resource is linked, project shows OK."""
reset_global_state()
proj_repo, link_repo, svc, sf = _setup_db()
proj = _create_project(proj_repo, "local/idx-project")