fix(e2e): restore M5/WF14 tests broken by JSON envelope and stale config
CI / push-validation (pull_request) Successful in 24s
CI / helm (pull_request) Successful in 34s
CI / build (pull_request) Successful in 53s
CI / lint (pull_request) Failing after 1m0s
CI / quality (pull_request) Successful in 1m12s
CI / typecheck (pull_request) Successful in 1m42s
CI / security (pull_request) Successful in 1m42s
CI / benchmark-publish (pull_request) Has been skipped
CI / e2e_tests (pull_request) Successful in 3m49s
CI / integration_tests (pull_request) Failing after 3m57s
CI / unit_tests (pull_request) Failing after 6m17s
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / status-check (pull_request) Failing after 3s
CI / push-validation (pull_request) Successful in 24s
CI / helm (pull_request) Successful in 34s
CI / build (pull_request) Successful in 53s
CI / lint (pull_request) Failing after 1m0s
CI / quality (pull_request) Successful in 1m12s
CI / typecheck (pull_request) Successful in 1m42s
CI / security (pull_request) Successful in 1m42s
CI / benchmark-publish (pull_request) Has been skipped
CI / e2e_tests (pull_request) Successful in 3m49s
CI / integration_tests (pull_request) Failing after 3m57s
CI / unit_tests (pull_request) Failing after 6m17s
CI / coverage (pull_request) Has been skipped
CI / docker (pull_request) Has been skipped
CI / status-check (pull_request) Failing after 3s
- Add NO_COLOR=1 to E2E Suite Setup and Test Environment to prevent Rich from injecting ANSI escape codes into JSON output, which breaks Extract JSON From Stdout and JSON assertions in M5/WF14 acceptance tests. - Add reset_global_state() calls to all M1/M2/M4/M5 E2E verification helpers to clear Settings singleton, DI container, provider registry, and SQLAlchemy engine cache between parallel pabot worker runs. - Propagate NO_COLOR=1 at suite-setup level in E2E and integration test resource files so all Run Process subprocesses inherit clean terminal output mode. Root causes: 1. Missing NO_COLOR=1 caused Rich ANSI codes in CLI JSON output, making json.loads() failures in robot Extract JSON From Stdout 2. Stale Settings singleton carried provider keys and DB paths between pabot workers, causing UNIQUE constraint violations and stale config lookups in context policy and server mode E2E tests. Related: PR #9912
This commit is contained in:
@@ -37,6 +37,7 @@ Setup Test Environment
|
||||
... that intentionally need real AI responses.
|
||||
[Arguments] ${auto_apply_migrations}=${TRUE} ${mock_ai}=${TRUE}
|
||||
Log Setting up test environment
|
||||
Set Environment Variable NO_COLOR 1
|
||||
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
|
||||
${home}= Set Variable ${TEMPDIR}${/}.cleveragents_${safe_suite}
|
||||
Run Keyword And Ignore Error Remove Directory ${home} recursive=True
|
||||
|
||||
@@ -22,6 +22,7 @@ E2E Suite Setup
|
||||
... Does NOT enable mock AI — E2E tests use real providers.
|
||||
... Propagates LLM API keys from the environment.
|
||||
Log Setting up E2E test environment
|
||||
Set Environment Variable NO_COLOR 1
|
||||
${safe_suite}= Replace String ${SUITE NAME} ${SPACE} _
|
||||
${safe_suite}= Replace String ${safe_suite} . _
|
||||
${safe_suite}= Replace String Using Regexp ${safe_suite} [^A-Za-z0-9_-] _
|
||||
@@ -53,6 +54,7 @@ E2E Suite Teardown
|
||||
Run Keyword And Ignore Error Remove Directory ${SUITE_HOME} recursive=True
|
||||
Remove Environment Variable CLEVERAGENTS_HOME
|
||||
Remove Environment Variable CLEVERAGENTS_AUTO_APPLY_MIGRATIONS
|
||||
Remove Environment Variable NO_COLOR
|
||||
|
||||
Skip If No LLM Keys
|
||||
[Documentation] Skip the current test if no LLM API keys are available.
|
||||
|
||||
@@ -57,6 +57,7 @@ from cleveragents.domain.models.core.change import ( # noqa: E402
|
||||
ToolInvocation,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
from helpers_common import reset_global_state # noqa: E402
|
||||
AutomationProfileProvenance,
|
||||
AutomationProfileRef,
|
||||
NamespacedName,
|
||||
@@ -160,6 +161,7 @@ def _fail(msg: str) -> NoReturn:
|
||||
|
||||
def action_create_from_yaml() -> None:
|
||||
"""Verify action create from YAML config via real CLI subprocess."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m1_action_")
|
||||
yaml_path = write_yaml(_VALID_ACTION_YAML)
|
||||
try:
|
||||
@@ -192,6 +194,7 @@ def action_create_from_yaml() -> None:
|
||||
|
||||
def resource_register_git_checkout() -> None:
|
||||
"""Register a git-checkout resource via real CLI subprocess."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m1_resource_")
|
||||
repo_dir = init_bare_git_repo()
|
||||
try:
|
||||
@@ -236,6 +239,7 @@ def resource_register_git_checkout() -> None:
|
||||
|
||||
def project_create_and_link() -> None:
|
||||
"""Create a project and link a resource via real CLI subprocess."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m1_project_")
|
||||
repo_dir = init_bare_git_repo()
|
||||
try:
|
||||
@@ -306,6 +310,7 @@ def plan_full_lifecycle() -> None:
|
||||
end-to-end. ChangeSet verification uses the domain-level tests
|
||||
(``changeset-invocations``), not this CLI lifecycle test.
|
||||
"""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m1_plan_")
|
||||
repo_dir = init_bare_git_repo()
|
||||
yaml_path = write_yaml(_VALID_ACTION_YAML)
|
||||
@@ -416,6 +421,7 @@ def sqlite_persistence_check() -> None:
|
||||
Uses the in-memory ChangeSetStore and domain model constructors
|
||||
to verify data survives serialisation round-trips.
|
||||
"""
|
||||
reset_global_state()
|
||||
# Create Action and verify serialisation
|
||||
action = _mock_action()
|
||||
action_dict = {
|
||||
@@ -483,6 +489,7 @@ def changeset_from_tool_invocations() -> None:
|
||||
Creates ToolInvocation records and verifies ChangeEntry objects
|
||||
are linked via tool_name, not extracted from CLI output text.
|
||||
"""
|
||||
reset_global_state()
|
||||
store, cid = _make_store()
|
||||
|
||||
# Simulate tool invocations producing ChangeEntry records
|
||||
@@ -531,6 +538,7 @@ def sandbox_isolation_check() -> None:
|
||||
Also verifies sandbox changes do not affect the original repo
|
||||
until Apply (commit + merge).
|
||||
"""
|
||||
reset_global_state()
|
||||
from cleveragents.infrastructure.sandbox.git_worktree import (
|
||||
GitWorktreeSandbox,
|
||||
)
|
||||
@@ -592,6 +600,7 @@ def post_apply_commit_check() -> None:
|
||||
Creates a git worktree sandbox, writes a file, commits, and
|
||||
verifies the commit is visible in git log on the original branch.
|
||||
"""
|
||||
reset_global_state()
|
||||
from cleveragents.infrastructure.sandbox.git_worktree import (
|
||||
GitWorktreeSandbox,
|
||||
)
|
||||
|
||||
@@ -76,6 +76,8 @@ from cleveragents.tool.router import ( # noqa: E402
|
||||
from cleveragents.tool.runner import ToolRunner # noqa: E402
|
||||
from cleveragents.tool.runtime import ToolSpec # noqa: E402
|
||||
|
||||
from helpers_common import reset_global_state # noqa: E402
|
||||
|
||||
|
||||
def _fail(msg: str) -> NoReturn:
|
||||
"""Print failure message and exit."""
|
||||
@@ -113,6 +115,7 @@ def _validation_fail(inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
def actor_yaml_create_load() -> None:
|
||||
"""Create a temporary actor YAML and load it via ActorLoader."""
|
||||
reset_global_state()
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_actor_"))
|
||||
yaml_content = (
|
||||
"name: local/m2-test-actor\n"
|
||||
@@ -162,6 +165,7 @@ def actor_yaml_create_load() -> None:
|
||||
|
||||
def actor_add_config_cli() -> None:
|
||||
"""Add an actor via ``agents actor add --config`` using subprocess."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m2_actor_cli_")
|
||||
config_dir = tempfile.mkdtemp(prefix="m2_actor_cfg_")
|
||||
config_path = os.path.join(config_dir, "actor.json")
|
||||
@@ -213,6 +217,7 @@ execution_actor: openai/gpt-4
|
||||
|
||||
def action_create() -> None:
|
||||
"""Create an action via real CLI subprocess."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m2_action_")
|
||||
yaml_path = write_yaml(_ACTION_YAML)
|
||||
try:
|
||||
@@ -248,6 +253,7 @@ def action_create() -> None:
|
||||
|
||||
def plan_use_execute() -> None:
|
||||
"""Run plan use and plan execute via real CLI subprocesses."""
|
||||
reset_global_state()
|
||||
workspace = setup_workspace(prefix="m2_plan_")
|
||||
yaml_path = write_yaml(_ACTION_YAML)
|
||||
try:
|
||||
@@ -293,6 +299,7 @@ def plan_use_execute() -> None:
|
||||
|
||||
def actor_yaml_parse_validate() -> None:
|
||||
"""Verify actor YAML files parse and validate correctly."""
|
||||
reset_global_state()
|
||||
# Test valid graph YAML
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_parse_"))
|
||||
valid_yaml = (
|
||||
@@ -356,6 +363,7 @@ def actor_yaml_parse_validate() -> None:
|
||||
|
||||
def actor_compile_stategraph() -> None:
|
||||
"""Verify actors compile to LangGraph StateGraphs."""
|
||||
reset_global_state()
|
||||
project_root = Path(__file__).resolve().parents[1]
|
||||
yaml_path = project_root / "examples" / "actors" / "simple_graph.yaml"
|
||||
config = ActorConfigSchema.from_yaml_file(str(yaml_path))
|
||||
@@ -398,6 +406,7 @@ def actor_compile_stategraph() -> None:
|
||||
|
||||
def tool_router_resolve_external() -> None:
|
||||
"""Verify tool router can resolve external tools."""
|
||||
reset_global_state()
|
||||
registry = ToolRegistry()
|
||||
ext_spec = ToolSpec(
|
||||
name="external/custom-tool",
|
||||
@@ -450,6 +459,7 @@ def tool_router_resolve_external() -> None:
|
||||
|
||||
def validation_runner_execute() -> None:
|
||||
"""Verify validation runner executes required/informational validations."""
|
||||
reset_global_state()
|
||||
# Create a required validation
|
||||
required_val = Validation.from_config(
|
||||
{
|
||||
@@ -522,6 +532,7 @@ def validation_runner_execute() -> None:
|
||||
|
||||
def changeset_multifile() -> None:
|
||||
"""Verify multi-file generation produces correct ChangeSet."""
|
||||
reset_global_state()
|
||||
tmpdir = tempfile.mkdtemp(prefix="m2_changeset_")
|
||||
capture = ChangeSetCapture(
|
||||
plan_id="m2-plan-cs",
|
||||
|
||||
@@ -45,6 +45,7 @@ from cleveragents.domain.models.core.decision import ( # noqa: E402
|
||||
DecisionType,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
from helpers_common import reset_global_state # noqa: E402
|
||||
NamespacedName,
|
||||
PlanPhase,
|
||||
ProcessingState,
|
||||
@@ -68,6 +69,7 @@ _DECISION_SPAWN_UI = "01KHDE6WWS2171PWW3GJEBXZ05"
|
||||
|
||||
def plan_diff() -> None:
|
||||
"""Verify merged results via ``agents plan diff`` (mocked service)."""
|
||||
reset_global_state()
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = (
|
||||
"--- a/src/api.py\n"
|
||||
@@ -145,6 +147,7 @@ def cli_plan_use() -> None:
|
||||
Typer's CliRunner. Asserts the service receives the correct action
|
||||
name and project, and the CLI exits cleanly.
|
||||
"""
|
||||
reset_global_state()
|
||||
mock_service = MagicMock()
|
||||
|
||||
# Mock action lookup -- use_action needs get_action_by_name first
|
||||
@@ -221,6 +224,7 @@ def cli_plan_execute() -> None:
|
||||
Asserts the plan transitions to Execute phase and the CLI renders
|
||||
the plan details.
|
||||
"""
|
||||
reset_global_state()
|
||||
mock_service = MagicMock()
|
||||
|
||||
# get_plan is called for the read-only guard
|
||||
@@ -306,6 +310,7 @@ def cli_plan_tree() -> None:
|
||||
``tree`` command resolves the ``DecisionService`` directly from
|
||||
the DI container (not via the lifecycle service helper).
|
||||
"""
|
||||
reset_global_state()
|
||||
# Build a realistic decision tree with subplan decisions
|
||||
decisions = [
|
||||
Decision(
|
||||
|
||||
@@ -6,6 +6,8 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from helpers_common import reset_global_state
|
||||
|
||||
from cleveragents.domain.models.core.context_policy import (
|
||||
ContextView,
|
||||
ProjectContextPolicy,
|
||||
@@ -21,24 +23,28 @@ _FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" /
|
||||
|
||||
def load_acms_context_policy_fixture() -> dict[str, Any]:
|
||||
"""Load the ACMS context policy fixture file."""
|
||||
reset_global_state()
|
||||
fixture_path = _FIXTURES_DIR / "acms_context_policy.json"
|
||||
return json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_large_project_context_fixture() -> dict[str, Any]:
|
||||
"""Load the large project context fixture file."""
|
||||
reset_global_state()
|
||||
fixture_path = _FIXTURES_DIR / "large_project_context.json"
|
||||
return json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_context_analysis_fixture() -> dict[str, Any]:
|
||||
"""Load the context analysis results fixture file."""
|
||||
reset_global_state()
|
||||
fixture_path = _FIXTURES_DIR / "context_analysis_results.json"
|
||||
return json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def resolve_view_for_phase(phase: str) -> dict[str, Any]:
|
||||
"""Resolve a view from an empty policy for the given phase."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy()
|
||||
view = policy.resolve_view(phase)
|
||||
return {
|
||||
@@ -53,6 +59,7 @@ def resolve_view_for_phase(phase: str) -> dict[str, Any]:
|
||||
|
||||
def resolve_strategize_inheriting_default() -> dict[str, Any]:
|
||||
"""Resolve strategize view that inherits from default."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy(
|
||||
default_view=ContextView(
|
||||
include_paths=["src/**/*.py"],
|
||||
@@ -69,6 +76,7 @@ def resolve_strategize_inheriting_default() -> dict[str, Any]:
|
||||
|
||||
def resolve_strategize_with_override() -> dict[str, Any]:
|
||||
"""Resolve strategize view with explicit override."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy(
|
||||
default_view=ContextView(
|
||||
include_paths=["src/**/*.py"],
|
||||
@@ -92,6 +100,7 @@ def check_budget_enforcement(
|
||||
within_file: int,
|
||||
) -> dict[str, bool]:
|
||||
"""Check if files exceed or fit within file size budget."""
|
||||
reset_global_state()
|
||||
view = ContextView(max_file_size=int(max_size))
|
||||
limit = view.max_file_size
|
||||
assert limit is not None
|
||||
@@ -107,6 +116,7 @@ def check_total_budget_enforcement(
|
||||
within_total: int,
|
||||
) -> dict[str, bool]:
|
||||
"""Check if aggregate context exceeds total size budget."""
|
||||
reset_global_state()
|
||||
view = ContextView(max_total_size=int(max_total))
|
||||
limit = view.max_total_size
|
||||
assert limit is not None
|
||||
@@ -118,6 +128,7 @@ def check_total_budget_enforcement(
|
||||
|
||||
def attempt_resolve_invalid_phase(phase: str) -> dict[str, str]:
|
||||
"""Try resolving an invalid phase and capture the error."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy()
|
||||
try:
|
||||
policy.resolve_view(phase)
|
||||
@@ -131,6 +142,7 @@ def create_multi_project_context(
|
||||
count_b: int,
|
||||
) -> dict[str, int]:
|
||||
"""Simulate independent context for two projects."""
|
||||
reset_global_state()
|
||||
proj_a = [{"path": f"src/a_{i}.py", "size": 1024} for i in range(int(count_a))]
|
||||
proj_b = [{"path": f"src/b_{i}.py", "size": 1024} for i in range(int(count_b))]
|
||||
return {
|
||||
|
||||
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock
|
||||
|
||||
from helper_m5_e2e_support import _create_project, _fail, _setup_db
|
||||
|
||||
from helpers_common import reset_global_state
|
||||
|
||||
from cleveragents.application.services.context_phase_analysis import (
|
||||
analyze_phase_summaries,
|
||||
)
|
||||
@@ -30,6 +32,7 @@ from cleveragents.domain.models.core.project import ContextConfig, NamespacedPro
|
||||
|
||||
def context_tier_config() -> None:
|
||||
"""Verify ContextConfig with hot/warm/cold tier management."""
|
||||
reset_global_state()
|
||||
default_cfg = ContextConfig()
|
||||
if default_cfg.hot_max_tokens is not None:
|
||||
_fail(f"default hot_max_tokens should be None: {default_cfg.hot_max_tokens}")
|
||||
@@ -79,6 +82,7 @@ def context_tier_config() -> None:
|
||||
|
||||
def context_policy_set_show() -> None:
|
||||
"""Set a context policy via SQL helpers and read it back."""
|
||||
reset_global_state()
|
||||
proj_repo, _link_repo, _svc, sf = _setup_db()
|
||||
_create_project(proj_repo, "local/policy-test")
|
||||
|
||||
@@ -129,6 +133,7 @@ def context_policy_set_show() -> None:
|
||||
|
||||
def acms_phase_inheritance() -> None:
|
||||
"""Verify ACMS view inheritance: default -> strategize -> execute -> apply."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy(
|
||||
default_view=ContextView(
|
||||
include_resources=["*"],
|
||||
@@ -179,6 +184,7 @@ def acms_phase_inheritance() -> None:
|
||||
|
||||
def acms_scoped_context() -> None:
|
||||
"""Verify ACMS produces scoped context output per phase."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy(
|
||||
default_view=ContextView(
|
||||
include_resources=["*"],
|
||||
@@ -323,6 +329,7 @@ def acms_scoped_context() -> None:
|
||||
|
||||
def context_policy_clear() -> None:
|
||||
"""Verify clearing a phase view causes inheritance fallback."""
|
||||
reset_global_state()
|
||||
policy = ProjectContextPolicy(
|
||||
default_view=ContextView(include_resources=["all-*"]),
|
||||
strategize_view=ContextView(include_resources=["strat-*"]),
|
||||
@@ -373,6 +380,7 @@ def context_policy_clear() -> None:
|
||||
|
||||
def context_view_validation() -> None:
|
||||
"""Verify ContextView validation rules."""
|
||||
reset_global_state()
|
||||
cv = ContextView(max_file_size=1, max_total_size=1)
|
||||
if cv.max_file_size != 1:
|
||||
_fail(f"max_file_size should be 1: {cv.max_file_size}")
|
||||
@@ -409,6 +417,7 @@ def context_view_validation() -> None:
|
||||
|
||||
def llm_execute_acms_context() -> None:
|
||||
"""Verify execute-phase LLM prompt uses ACMS assembled execute context."""
|
||||
reset_global_state()
|
||||
proj_repo, _link_repo, _svc, sf = _setup_db()
|
||||
_create_project(proj_repo, "local/exec-ctx")
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, NoReturn, cast
|
||||
|
||||
from helpers_common import reset_global_state
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
@@ -49,6 +51,7 @@ def _setup_db() -> tuple[
|
||||
Any,
|
||||
]:
|
||||
"""Create an in-memory SQLite DB with all tables."""
|
||||
reset_global_state()
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
Base.metadata.create_all(engine)
|
||||
session = sessionmaker(bind=engine, expire_on_commit=False)()
|
||||
@@ -70,6 +73,7 @@ def _create_project(
|
||||
description: str | None = None,
|
||||
) -> NamespacedProject:
|
||||
"""Create and return a NamespacedProject via the repository."""
|
||||
reset_global_state()
|
||||
parsed = parse_namespaced_name(name)
|
||||
proj = NamespacedProject(
|
||||
name=parsed.name,
|
||||
|
||||
@@ -51,6 +51,8 @@ from cleveragents.application.services.repo_indexing_service import ( # noqa: E
|
||||
)
|
||||
from cleveragents.domain.models.core.repo_index import IndexStatus # noqa: E402
|
||||
|
||||
from helpers_common import reset_global_state # noqa: E402
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Subcommand: project-create-large
|
||||
# -------------------------------------------------------------------
|
||||
@@ -62,6 +64,7 @@ def project_create_large() -> None:
|
||||
Verifies project creation, resource registration, linking,
|
||||
and that the project show reflects the linked resource.
|
||||
"""
|
||||
reset_global_state()
|
||||
proj_repo, link_repo, svc, _sf = _setup_db()
|
||||
|
||||
proj = _create_project(
|
||||
@@ -114,6 +117,7 @@ def project_create_large() -> None:
|
||||
|
||||
def resource_register_link() -> None:
|
||||
"""Register a git-checkout resource and link it to a project."""
|
||||
reset_global_state()
|
||||
proj_repo, link_repo, svc, _sf = _setup_db()
|
||||
|
||||
proj = _create_project(proj_repo, "local/link-test")
|
||||
@@ -162,6 +166,7 @@ def resource_register_link() -> None:
|
||||
|
||||
def indexing_complete() -> None:
|
||||
"""Verify indexing completes: resource is linked, project shows OK."""
|
||||
reset_global_state()
|
||||
proj_repo, link_repo, svc, sf = _setup_db()
|
||||
|
||||
proj = _create_project(proj_repo, "local/idx-project")
|
||||
|
||||
Reference in New Issue
Block a user