diff --git a/CHANGELOG.md b/CHANGELOG.md index 309d097c2..b73240a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ guidance propagation -- and verify cautious-profile confidence-threshold pausing (S37262-37367) including a pause-and-resume flow. Facade stub updated to echo guidance text. (#961) +- Added TDD bug-capture tests for bug #1023: CLI commands fail without explicit + `agents init` when `CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true` is set. Two + Behave BDD scenarios and two Robot Framework integration tests verify that + `resource add` and `project create` succeed in a fresh environment without + prior init. Tests use `@tdd_expected_fail` until the bug fix is merged. + (#1033) - Added TDD bug-capture tests for bug #1076 — `use_action()` does not propagate `automation_profile` to Plan. Three Behave BDD scenarios (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence diff --git a/features/steps/tdd_e2e_implicit_init_steps.py b/features/steps/tdd_e2e_implicit_init_steps.py new file mode 100644 index 000000000..640d80c3a --- /dev/null +++ b/features/steps/tdd_e2e_implicit_init_steps.py @@ -0,0 +1,173 @@ +"""Step definitions for TDD Bug #1023 — implicit init requirement. + +These steps exercise real CLI commands in a fresh environment where +``agents init`` has NOT been explicitly run. The environment variable +``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` is set, which *should* +cause the database to be created implicitly. + +Bug #1023 reports that ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` +triggers schema migrations on an existing database but does NOT create +the database file or its parent directory structure. As a result, any +CLI command that touches the database (e.g., ``resource add``, +``project create``) fails with: + + sqlite3.OperationalError: unable to open database file + +The assertions here expect the commands to succeed (exit code 0), +proving implicit init works and preventing regressions. + +Mock strategy +~~~~~~~~~~~~~ +No mocks are needed — we exercise the real CLI via Typer's CliRunner +in a pristine temporary directory. We only manipulate environment +variables to create a fresh, uninitialised CLEVERAGENTS_HOME. We +remove ``CLEVERAGENTS_DATABASE_URL``, ``CLEVERAGENTS_TEST_DATABASE_URL``, +``BEHAVE_TESTING``, and ``CLEVERAGENTS_TEMPLATE_DB`` so the container +derives the DB path from ``CLEVERAGENTS_HOME``, migration prompts are +not auto-approved by the testing guard, and the template-DB fast-path +does not silently create the database for us. +""" + +from __future__ import annotations + +import os +import shutil +import tempfile + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.application.container import reset_container +from cleveragents.cli.main import app + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# Given steps +# --------------------------------------------------------------------------- + + +@given("a fresh isolated environment for tdd-implicit-init") +def step_fresh_environment(context: Context) -> None: + """Set up a fresh temporary environment with no existing database. + + Creates a temporary directory to serve as ``CLEVERAGENTS_HOME``. + Does NOT run ``agents init`` — the entire point of this TDD test is + to verify that CLI commands work without it. + + Removes ``BEHAVE_TESTING`` so the migration runner's auto-approve + guard does not mask the real code path. + """ + tmpdir = tempfile.mkdtemp(prefix="tdd_implicit_init_") + context.tdd_implicit_init_tmpdir = tmpdir + + # Save and remove env vars that point to pre-migrated databases, + # template-DB fast-paths, or testing guards that auto-approve + # migration prompts. CLEVERAGENTS_TEMPLATE_DB is particularly + # important: when set, the Behave template-DB patch copies a + # pre-migrated database, silently bypassing the directory-creation + # bug this test is designed to capture. + saved_env: dict[str, str | None] = { + k: os.environ.pop(k, None) + for k in ( + "CLEVERAGENTS_AUTO_APPLY_MIGRATIONS", + "CLEVERAGENTS_DATABASE_URL", + "CLEVERAGENTS_TEST_DATABASE_URL", + "CLEVERAGENTS_HOME", + "BEHAVE_TESTING", + "CLEVERAGENTS_TEMPLATE_DB", + "CLEVERAGENTS_TESTING_USE_MOCK_AI", + ) + } + context.tdd_implicit_init_saved_env = saved_env + + # Point CLEVERAGENTS_HOME at our pristine temp directory. + os.environ["CLEVERAGENTS_HOME"] = tmpdir + + # Force an isolated SQLite database path under this scenario's temp + # directory. This avoids leaking state from repository-level local + # database files created by prior test runs. + db_url = f"sqlite:///{tmpdir}/db/cleveragents.db" + os.environ["CLEVERAGENTS_DATABASE_URL"] = db_url + os.environ["CLEVERAGENTS_TEST_DATABASE_URL"] = db_url + + # Reset the DI container so it picks up our env overrides. + reset_container() + + def _cleanup() -> None: + for key, val in saved_env.items(): + if val is not None: + os.environ[key] = val + else: + os.environ.pop(key, None) + reset_container() + shutil.rmtree(tmpdir, ignore_errors=True) + + context.add_cleanup(_cleanup) + + +@given("CLEVERAGENTS_AUTO_APPLY_MIGRATIONS is set to true") +def step_set_auto_apply_migrations(context: Context) -> None: + """Ensure CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true is in the environment. + + This is the env var that should, in theory, trigger implicit + database creation and migration. Bug #1023 reports it does not + create the directory structure. + """ + os.environ["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when('I run the CLI command "{command}" without prior init') +def step_run_cli_command_without_init(context: Context, command: str) -> None: + """Invoke a CLI command via CliRunner without running ``agents init`` first. + + The command string is split into tokens and passed to the main CLI + app. ``CLEVERAGENTS_HOME`` is set to the pristine temp directory + and ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` is in the + environment. + + Exceptions are caught by CliRunner (the default behaviour) so that + the result object always carries an ``exit_code`` and ``output``. + This is essential for ``@tdd_expected_fail`` inversion — if + exceptions propagated as non-AssertionError, the inversion guard + in ``apply_tdd_inversion`` would skip inversion and the scenario + would be reported as a hard failure instead of an expected failure. + """ + # Reset container again to ensure a clean UoW derivation. + reset_container() + + args = command.split() + context.tdd_implicit_init_result = runner.invoke(app, args) + + +# --------------------------------------------------------------------------- +# Then steps +# --------------------------------------------------------------------------- + + +@then("the tdd-implicit-init command should exit with code {code:d}") +def step_exit_code(context: Context, code: int) -> None: + """Assert the CLI command exited with the expected return code.""" + result = context.tdd_implicit_init_result + assert result is not None, "CLI command was not invoked" + actual = result.exit_code + assert actual == code, ( + f"Expected exit code {code}, got {actual}.\nOutput:\n{result.output}" + ) + + +@then('the tdd-implicit-init command output should contain "{text}"') +def step_output_contains(context: Context, text: str) -> None: + """Assert the CLI command output contains the expected text.""" + result = context.tdd_implicit_init_result + assert result is not None, "CLI command was not invoked" + assert text in result.output, ( + f"Expected output to contain {text!r}.\nActual output:\n{result.output}" + ) diff --git a/features/tdd_e2e_implicit_init.feature b/features/tdd_e2e_implicit_init.feature new file mode 100644 index 000000000..58c2a2b6a --- /dev/null +++ b/features/tdd_e2e_implicit_init.feature @@ -0,0 +1,35 @@ +@tdd_expected_fail @tdd_issue @tdd_issue_1023 +Feature: TDD Bug #1023 — CLI commands should succeed without explicit init + As a developer + I want to verify that CLI commands that touch the database succeed + without requiring an explicit `agents init` invocation first + So that the bug is captured and will be caught by a regression test + + Bug #1023 reports that running any DB-dependent CLI command (e.g., + `resource add`, `project create`) in a fresh environment fails with + `sqlite3.OperationalError: unable to open database file` even when + `CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true` is set. The specification + implies that initialization should happen implicitly. + + The root cause is that `CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true` + triggers schema migrations on an existing database but does NOT create + the database file or its parent directory structure. A manual + `agents init --yes --force` is required first. + + These tests assert the expected behaviour (implicit init) and will + fail until the bug is fixed. The @tdd_expected_fail tag inverts the + result so CI remains green while the defect is open. + + Scenario: Resource add succeeds in a fresh environment without explicit init + Given a fresh isolated environment for tdd-implicit-init + And CLEVERAGENTS_AUTO_APPLY_MIGRATIONS is set to true + When I run the CLI command "resource add git-checkout local/tdd-test-resource --path . --branch main" without prior init + Then the tdd-implicit-init command should exit with code 0 + And the tdd-implicit-init command output should contain "tdd-test-resource" + + Scenario: Project create succeeds in a fresh environment without explicit init + Given a fresh isolated environment for tdd-implicit-init + And CLEVERAGENTS_AUTO_APPLY_MIGRATIONS is set to true + When I run the CLI command "project create local/tdd-test-project" without prior init + Then the tdd-implicit-init command should exit with code 0 + And the tdd-implicit-init command output should contain "tdd-test-project" diff --git a/robot/helper_tdd_e2e_implicit_init.py b/robot/helper_tdd_e2e_implicit_init.py new file mode 100644 index 000000000..55245feb8 --- /dev/null +++ b/robot/helper_tdd_e2e_implicit_init.py @@ -0,0 +1,181 @@ +"""Helper script for tdd_e2e_implicit_init.robot smoke tests. + +Each subcommand exercises a real CLI command in a fresh environment +where ``agents init`` has NOT been run, to reproduce bug #1023. + +Bug #1023 reports that ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` +triggers schema migrations on an existing database but does NOT create +the database file or its parent directory structure. As a result, any +CLI command that touches the database fails with: + + sqlite3.OperationalError: unable to open database file + +The helper reports the **real** outcome: it exits 0 and prints the +sentinel when the operation succeeds (bug is fixed), and exits 1 when +the bug is still present. The ``tdd_expected_fail_listener`` on the +Robot side handles pass/fail inversion while the bug remains open. + +Mock strategy +~~~~~~~~~~~~~ +No mocks are needed — we exercise the real CLI via Typer's CliRunner +in a pristine temporary directory. We only manipulate environment +variables to create a fresh, uninitialised CLEVERAGENTS_HOME and set +``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true``. +""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +# Ensure local source tree is importable +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.application.container import reset_container # noqa: E402 +from cleveragents.cli.main import app # noqa: E402 + +runner = CliRunner() + + +def _run_cli_without_init(args: list[str], sentinel: str) -> None: + """Run a CLI command in a fresh environment without prior init. + + Creates a pristine temp directory, sets CLEVERAGENTS_HOME to it, + enables CLEVERAGENTS_AUTO_APPLY_MIGRATIONS, and invokes the CLI. + Exits 0 with sentinel on success; exits 1 on failure. + + Environment variables are always restored in the ``finally`` block + so subsequent tests are not polluted even if the CLI invocation + raises an unexpected exception. + + Exceptions are caught by CliRunner (the default behaviour) so that + the result object always carries an ``exit_code`` and ``output``. + Using ``catch_exceptions=False`` would cause non-AssertionError + exceptions to propagate, preventing the ``tdd_expected_fail`` + listener from inverting the result. + + Parameters + ---------- + args: + CLI arguments to pass (e.g., ``["resource", "add", ...]``). + sentinel: + Sentinel string to print on success. + """ + tmpdir = tempfile.mkdtemp(prefix="tdd_implicit_init_robot_") + env_save: dict[str, str | None] = {} + try: + # Save and remove env vars that point to pre-migrated databases, + # template-DB fast-paths, or testing guards that auto-approve + # migration prompts. CLEVERAGENTS_TEMPLATE_DB is particularly + # important: when set, the Behave template-DB patch copies a + # pre-migrated database, silently bypassing the directory- + # creation bug this test is designed to capture. + for key in ( + "CLEVERAGENTS_AUTO_APPLY_MIGRATIONS", + "CI", + "BEHAVE_TESTING", + "ROBOT_TESTING", + "CLEVERAGENTS_HOME", + "CLEVERAGENTS_DATABASE_URL", + "CLEVERAGENTS_TEST_DATABASE_URL", + "CLEVERAGENTS_TEMPLATE_DB", + "CLEVERAGENTS_TESTING_USE_MOCK_AI", + ): + env_save[key] = os.environ.pop(key, None) + + os.environ["CLEVERAGENTS_HOME"] = tmpdir + os.environ["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" + + # Reset DI container so it picks up our env overrides. + reset_container() + + result = runner.invoke(app, args) + + if result.exit_code != 0: + print( + f"CLI command failed with exit code {result.exit_code}", + file=sys.stderr, + ) + print(f"Output: {result.output}", file=sys.stderr) + sys.exit(1) + + # Bug fixed — command succeeded without explicit init. + print(sentinel) + finally: + # Always restore env vars and clean up the temp directory, + # even if runner.invoke() or assertions raise. + for key, val in env_save.items(): + if val is not None: + os.environ[key] = val + else: + os.environ.pop(key, None) + reset_container() + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def resource_add_no_init() -> None: + """Invoke ``resource add`` without prior ``agents init``. + + Exercises the ``resource add git-checkout`` command in a fresh + environment. If the database directory structure is not + auto-created, this will fail with OperationalError. + """ + _run_cli_without_init( + [ + "resource", + "add", + "git-checkout", + "local/tdd-test-resource", + "--path", + ".", + "--branch", + "main", + ], + "tdd-resource-add-no-init-ok", + ) + + +def project_create_no_init() -> None: + """Invoke ``project create`` without prior ``agents init``. + + Exercises the ``project create`` command in a fresh environment. + If the database directory structure is not auto-created, this + will fail with OperationalError. + """ + _run_cli_without_init( + ["project", "create", "local/tdd-test-project"], + "tdd-project-create-no-init-ok", + ) + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], None]] = { + "resource-add-no-init": resource_add_no_init, + "project-create-no-init": project_create_no_init, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print( + f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>", + file=sys.stderr, + ) + sys.exit(1) + cmd = _COMMANDS[sys.argv[1]] + cmd() diff --git a/robot/tdd_e2e_implicit_init.robot b/robot/tdd_e2e_implicit_init.robot new file mode 100644 index 000000000..778f120b0 --- /dev/null +++ b/robot/tdd_e2e_implicit_init.robot @@ -0,0 +1,45 @@ +*** Settings *** +Documentation TDD Bug #1023 — CLI commands should succeed without explicit init +... Integration smoke tests verifying that DB-dependent CLI commands +... (e.g., ``resource add``, ``project create``) succeed in a fresh +... environment where ``agents init`` has NOT been run, when +... ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` is set. +... +... Bug #1023 reports that ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` +... triggers migrations on an existing database but does NOT create +... the database file or its parent directory structure. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_tdd_e2e_implicit_init.py + +*** Test Cases *** +TDD Resource Add Succeeds Without Explicit Init + [Documentation] Verify that ``resource add`` succeeds in a fresh + ... environment without prior ``agents init`` when + ... ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` is set. + ... The helper exits 0 with a sentinel when the command + ... succeeds (bug is fixed), and exits 1 when the bug is + ... present (OperationalError). + [Tags] tdd_bug tdd_bug_1023 + ${result}= Run Process ${PYTHON} ${HELPER} resource-add-no-init cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-resource-add-no-init-ok + +TDD Project Create Succeeds Without Explicit Init + [Documentation] Verify that ``project create`` succeeds in a fresh + ... environment without prior ``agents init`` when + ... ``CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true`` is set. + ... The helper exits 0 with a sentinel when the command + ... succeeds (bug is fixed), and exits 1 when the bug is + ... present (OperationalError). + [Tags] tdd_bug tdd_bug_1023 + ${result}= Run Process ${PYTHON} ${HELPER} project-create-no-init cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-project-create-no-init-ok