From 4fff7f67c5b28bc8e6dfd5357b5296164bbd8f0f Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Tue, 10 Mar 2026 22:27:37 +0000 Subject: [PATCH] fix(test): use scoped_session to prevent GC-induced data loss in r2cov MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite in-memory engines use SingletonThreadPool, giving every Session the same underlying connection. Repository methods create a new Session via the factory, flush, then return — letting the Session go out of scope. Under high memory pressure (e.g. 32 parallel behave workers) Python's garbage collector closes these orphaned Sessions, issuing an implicit ROLLBACK on the shared connection and wiping flushed-but-uncommitted rows written by other Sessions. Replace the plain sessionmaker with scoped_session in the r2cov Background step so that every factory() call returns the same Session instance. A single long-lived Session per scenario eliminates the premature close/rollback window entirely. Verified: 3 consecutive green runs with --processes 32 (10 099 scenarios, 0 failures each). Refs: #570 --- .../steps/repositories_coverage_r2_steps.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/features/steps/repositories_coverage_r2_steps.py b/features/steps/repositories_coverage_r2_steps.py index 149f8f014..296bd933f 100644 --- a/features/steps/repositories_coverage_r2_steps.py +++ b/features/steps/repositories_coverage_r2_steps.py @@ -18,7 +18,7 @@ from behave import given, then, when from behave.runner import Context from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session, scoped_session, sessionmaker from cleveragents.core.exceptions import DatabaseError from cleveragents.domain.models.core.checkpoint import Checkpoint, CheckpointMetadata @@ -234,7 +234,20 @@ def step_fresh_db(context: Context) -> None: engine = create_engine("sqlite:///:memory:", echo=False) Base.metadata.create_all(engine) context.r2_engine = engine - context.r2_session_factory = sessionmaker(bind=engine) + # Use scoped_session so that every ``factory()`` call within the + # same thread returns the *same* Session instance. With plain + # ``sessionmaker``, each ``factory()`` call creates a new Session. + # SQLite in-memory uses ``SingletonThreadPool`` (one connection per + # thread), so all sessions share the same connection. When a + # session created inside a repository method goes out of scope, + # Python's garbage collector may close it, issuing an implicit + # ROLLBACK on the shared connection — wiping flushed-but-uncommitted + # rows written by *other* sessions. Under high memory pressure + # (e.g. 32 parallel worker processes) GC fires often enough to + # cause intermittent data loss between ``flush()`` and ``commit()``. + # ``scoped_session`` avoids the problem entirely: one Session lives + # for the whole scenario, so there is no premature close/rollback. + context.r2_session_factory = scoped_session(sessionmaker(bind=engine)) # Pre-create repos used by multiple scenarios context.r2_skill_repo = SkillRepository(