From a8f7ed57cbca278a0daaabfea380936e2d8a714c Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Sun, 1 Mar 2026 22:18:45 +0000 Subject: [PATCH] perf(tests): reduce per-feature startup cost with shared fixtures and lazy imports Created scripts/create_template_db.py that builds a pre-migrated SQLite template database using Base.metadata.create_all() + alembic stamp (~5ms for 34 tables, vs ~0.5-3s x 25 Alembic migrations per scenario). Nox unit_tests and coverage_report sessions generate the template before test execution and propagate CLEVERAGENTS_TEMPLATE_DB env var to all workers. features/environment.py before_all() installs a monkey-patch on MigrationRunner.init_or_upgrade that copies the template for fresh scenario temp DBs, falling through to real migrations for :memory:, existing files, and migration-runner unit tests. Quick wins: sleep(0.5) -> sleep(0.05) in cli_streaming wait step; removed redundant Background re-declaration in cli_streaming.feature scenario 7. ISSUES CLOSED: #483 --- CHANGELOG.md | 6 +++ features/cli_streaming.feature | 3 -- features/environment.py | 65 +++++++++++++++++++++++++ features/steps/cli_streaming_steps.py | 2 +- noxfile.py | 28 +++++++++++ scripts/create_template_db.py | 68 +++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 scripts/create_template_db.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c82c0c8f1..49265e97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Added pre-migrated SQLite template database via `scripts/create_template_db.py` to eliminate + repeated Alembic migrations per BDD scenario. Nox sessions propagate the template via + `CLEVERAGENTS_TEMPLATE_DB` env var; `features/environment.py` monkey-patches + `MigrationRunner.init_or_upgrade` to copy the template for fresh scenario temp DBs, falling + through to real migrations for `:memory:`, existing files, and migration-runner unit + tests. (#483) - Replaced coverage.py (sys.settrace) with slipcover (bytecode instrumentation) for faster coverage collection. Each behave-parallel worker now produces per-feature JSON coverage files; slipcover --merge combines them. CI workflow JSON key lookups handle both slipcover and diff --git a/features/cli_streaming.feature b/features/cli_streaming.feature index 5a60fc069..cda3e47f4 100644 --- a/features/cli_streaming.feature +++ b/features/cli_streaming.feature @@ -50,9 +50,6 @@ Feature: CLI Streaming Integration And timing should be sequential Scenario: Non-streaming mode is faster for simple commands - Given a temporary directory - And I initialize a new project named "streaming-test" - And I add a test file to the project When I measure time for tell command "simple change" without streaming And I measure time for tell command "another simple change" with streaming Then both commands should complete successfully diff --git a/features/environment.py b/features/environment.py index 0d3badb3c..8984cfe3d 100644 --- a/features/environment.py +++ b/features/environment.py @@ -6,6 +6,7 @@ import shutil import sys import tempfile from pathlib import Path +from typing import Any LANGSMITH_ENV_VARS = [ "CLEVERAGENTS_LANGSMITH_ENABLED", @@ -56,6 +57,70 @@ def before_all(context): except ImportError: pass # Container not needed for all tests + # --- Template-DB fast-path --- + # When CLEVERAGENTS_TEMPLATE_DB is set (by nox sessions), monkey-patch + # MigrationRunner.init_or_upgrade so that fresh file-based SQLite + # databases are created by copying the pre-migrated template (~1ms) + # instead of running 25 Alembic migrations (~0.5-3s each). + _install_template_db_patch() + + +def _install_template_db_patch() -> None: + """Monkey-patch MigrationRunner to copy a template DB for fresh SQLite files.""" + template_path = os.environ.get("CLEVERAGENTS_TEMPLATE_DB") + if not template_path or not Path(template_path).is_file(): + return + + try: + from cleveragents.infrastructure.database.migration_runner import ( + MigrationRunner, + ) + except ImportError: + return + + _original_init_or_upgrade = MigrationRunner.init_or_upgrade + + # Prefixes used by before_scenario when creating per-scenario temp DBs. + _SCENARIO_DB_PREFIXES = ("cleveragents_", "cleveragents_test_") + + def _fast_init_or_upgrade(self: Any, **kwargs: Any) -> None: + """Copy the template DB instead of running Alembic migrations. + + Falls through to the original method for: + - Non-SQLite databases + - In-memory SQLite databases (:memory:) + - SQLite files that already exist on disk + - SQLite files whose basename doesn't match the before_scenario + temp-file naming pattern (cleveragents_* / cleveragents_test_*) + """ + db_url: str = getattr(self, "database_url", "") + + # Only intercept file-based SQLite URLs + if not db_url.startswith("sqlite") or ":memory:" in db_url: + return _original_init_or_upgrade(self, **kwargs) + + # Extract the file path from the URL + db_file_path = db_url.replace("sqlite:///", "") + if not db_file_path.startswith("/"): + db_file_path = "/" + db_file_path + + db_path = Path(db_file_path) + + # Only apply to scenario-generated temp DBs (avoid hijacking + # migration-runner unit tests that use custom URLs). + if not any(db_path.name.startswith(p) for p in _SCENARIO_DB_PREFIXES): + return _original_init_or_upgrade(self, **kwargs) + + # Only copy template for databases that don't exist yet (fresh scenario) + if db_path.exists(): + return _original_init_or_upgrade(self, **kwargs) + + # Copy the template — creates a fully-migrated DB in ~1ms + db_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(template_path, db_file_path) + + MigrationRunner.init_or_upgrade = _fast_init_or_upgrade # type: ignore[assignment] + def before_scenario(context, scenario): """Set up before each scenario.""" diff --git a/features/steps/cli_streaming_steps.py b/features/steps/cli_streaming_steps.py index 3cfc0611c..d95e49427 100644 --- a/features/steps/cli_streaming_steps.py +++ b/features/steps/cli_streaming_steps.py @@ -311,7 +311,7 @@ def step_measure_time_with_streaming(context, prompt): @when("I wait for completion") def step_wait_for_completion(context): """Wait for command completion.""" - time.sleep(0.5) + time.sleep(0.05) @when("I send interrupt signal after {seconds:d} seconds") diff --git a/noxfile.py b/noxfile.py index 578de4ae8..8a7c0bdd2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -69,6 +69,24 @@ def _split_pabot_args(posargs: list[str]) -> tuple[list[str], list[str]]: return pabot_args, robot_args +def _create_template_db(session: nox.Session) -> str: + """Build the pre-migrated SQLite template and return its path. + + Runs ``scripts/create_template_db.py`` to produce a SQLite file with all + tables already created via ``Base.metadata.create_all()`` and the + alembic_version table stamped at HEAD. Each test scenario can then + ``shutil.copy`` this file instead of running 25 Alembic migrations. + """ + template_path = str(Path("build/.template-migrated.db").resolve()) + session.run( + "python", + "scripts/create_template_db.py", + template_path, + silent=True, + ) + return template_path + + def _install_behave_parallel(session: nox.Session) -> None: """Install behave-parallel with a custom CLI wrapper.""" @@ -356,6 +374,11 @@ def unit_tests(session: nox.Session): session.install("-e", ".[tests]") _install_behave_parallel(session) + # Build a pre-migrated template DB so each scenario can copy it + # instead of running 25 Alembic migrations from scratch. + template_path = _create_template_db(session) + session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path + behave_cmd = session.bin + "/behave-parallel" parallel_args = _behave_parallel_args(session.posargs) @@ -519,6 +542,11 @@ def coverage_report(session: nox.Session): os.makedirs("build", exist_ok=True) + # Build a pre-migrated template DB so each scenario can copy it + # instead of running 25 Alembic migrations from scratch. + template_path = _create_template_db(session) + session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path + session.env["PYTHONPATH"] = str(Path("src").resolve()) session.env["NO_COLOR"] = "1" source_paths = "src" diff --git a/scripts/create_template_db.py b/scripts/create_template_db.py new file mode 100644 index 000000000..46b662df9 --- /dev/null +++ b/scripts/create_template_db.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Create a pre-migrated template SQLite database for fast test setup. + +Instead of running 25 Alembic migrations per scenario (~0.5-3s each), +tests can copy this template file (~1ms) and get an identical schema. + +Usage: + python scripts/create_template_db.py [output_path] + +Default output: build/.template-migrated.db +""" + +import sys +from pathlib import Path + +# Ensure src/ is importable +src_dir = Path(__file__).resolve().parent.parent / "src" +sys.path.insert(0, str(src_dir)) + + +def create_template(output_path: str = "build/.template-migrated.db") -> None: + """Create a fully-migrated template SQLite database. + + Uses Base.metadata.create_all() to create all tables in a single DDL + batch (~5ms), then stamps the alembic_version table with the head + revision so MigrationRunner sees no pending migrations. + """ + from alembic import command + from alembic.config import Config + from alembic.script import ScriptDirectory + from sqlalchemy import create_engine + + from cleveragents.infrastructure.database.models import Base + + out = Path(output_path) + out.parent.mkdir(parents=True, exist_ok=True) + + # Remove existing template so we always create fresh + if out.exists(): + out.unlink() + + db_url = f"sqlite:///{out.resolve()}" + engine = create_engine(db_url, connect_args={"check_same_thread": False}) + + # Create all 33 tables in one fast DDL batch + Base.metadata.create_all(engine) + + # Stamp alembic_version with the head revision so MigrationRunner + # sees the database as fully migrated (no pending migrations). + alembic_ini = Path(__file__).resolve().parent.parent / "alembic.ini" + cfg = Config(str(alembic_ini)) + sd = ScriptDirectory.from_config(cfg) + head = sd.get_current_head() + if head is None: + msg = "No Alembic revisions found — cannot stamp template database." + raise RuntimeError(msg) + + with engine.connect() as conn: + cfg.attributes["connection"] = conn + command.stamp(cfg, head) + conn.commit() + + engine.dispose() + + +if __name__ == "__main__": + path = sys.argv[1] if len(sys.argv) > 1 else "build/.template-migrated.db" + create_template(path)