From 4e3bf7d3ad61c6a10b2e135fe37c47551a8e7ebb Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Fri, 6 Mar 2026 20:28:37 +0000 Subject: [PATCH] test(cli): add failing tests for agents init --yes missing option Add TDD-style Behave BDD tests for the missing agents init --yes flag (bug #522). Five Gherkin scenarios cover: exit code validation, prompt suppression, -y alias, output summary fields, and interactive-mode regression guard. Includes Robot Framework smoke tests (tagged @wip) and ASV benchmarks. Configure behave.ini to exclude @wip scenarios globally and noxfile.py to exclude wip-tagged Robot suites, so TDD-failing tests do not break CI. Review feedback addressed: - Remove unnecessary # type: ignore from benchmark (outside Pyright scope) - Fix Then...Then to Then...And in Gherkin (L1) - Fix CHANGELOG 'three scenarios' to 'five scenarios' (L2) - Add behave.ini documentation for @wip workaround (Aditya F1) - Rename Scenario 2 title to 'suppresses interactive prompts' (Aditya F2) Closes #536 --- CHANGELOG.md | 5 + behave.ini | 6 + benchmarks/cli_init_yes_bench.py | 118 ++++++++++++ features/cli_init_yes_flag.feature | 56 ++++++ features/steps/cli_init_yes_flag_steps.py | 210 ++++++++++++++++++++++ noxfile.py | 2 + robot/cli_init_yes_flag.robot | 52 ++++++ 7 files changed, 449 insertions(+) create mode 100644 benchmarks/cli_init_yes_bench.py create mode 100644 features/cli_init_yes_flag.feature create mode 100644 features/steps/cli_init_yes_flag_steps.py create mode 100644 robot/cli_init_yes_flag.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index e2bb52577..69739ed08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Added TDD-style failing Behave BDD tests for the missing `agents init --yes` flag. + Five scenarios: four TDD-failing tests (exit code, prompt suppression, `-y` alias, + output summary) and one regression guard for interactive mode. Includes Robot + Framework smoke tests and ASV benchmarks. Tests are intentionally failing until + the bug fix for #522 is applied. (#536) - Added general-purpose domain event system under `cleveragents.infrastructure.events`. `EventType` StrEnum defines 38 typed event identifiers across 9 domains (plan lifecycle, decision, invariant, actor, diff --git a/behave.ini b/behave.ini index b19e710cd..e76b61f9c 100644 --- a/behave.ini +++ b/behave.ini @@ -1,4 +1,10 @@ [behave] paths = features +# Exclude @wip scenarios globally so TDD failing tests do not break CI. +# Any contributor tagging a scenario @wip will have it skipped by default. +# NOTE: --tags=@wip on the CLI will NOT work; Behave ANDs ini and CLI tags. +# To run a @wip scenario locally, target it by file/line number: +# behave features/.feature: +tags = ~@wip stdout_capture = no stderr_capture = no diff --git a/benchmarks/cli_init_yes_bench.py b/benchmarks/cli_init_yes_bench.py new file mode 100644 index 000000000..4f2be2ecd --- /dev/null +++ b/benchmarks/cli_init_yes_bench.py @@ -0,0 +1,118 @@ +"""ASV benchmarks for ``agents init --yes`` invocation time. + +Measures in-process execution time for the init command with the --yes flag, +which should perform non-interactive initialization using defaults. + +These benchmarks target bug #522 and are expected to error until the fix +is applied (the --yes flag does not yet exist). +""" + +from __future__ import annotations + +import shutil +import sys +import tempfile +from pathlib import Path +from unittest.mock import create_autospec, patch + +try: + from cleveragents.cli.main import app +except ModuleNotFoundError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cleveragents.cli.main import app + +from typer.testing import CliRunner + +from cleveragents.application.services.project_service import ProjectService + + +class _MockProject: + """Lightweight stand-in for the Project model with spec fields. + + We cannot use ``create_autospec(Project)`` because the spec-required + output fields (``data_dir``, ``config_path``, ``database_status``, + ``directories``) do not yet exist on the legacy ``Project`` model. + """ + + name: str + path: Path + data_dir: Path + config_path: Path + database_status: str + directories: list[str] + + +class InitYesFlagSuite: + """Benchmark ``agents init --yes`` invocation.""" + + timeout = 30.0 + + def setup(self) -> None: + self._runner = CliRunner() + self._tmpdir = tempfile.mkdtemp() + self._last_exit_code = -1 + + def teardown(self) -> None: + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def _invoke_with_mock(self, args: list[str], project_name: str) -> None: + """Invoke the CLI with a mocked container and the given *args*. + + Stores the result exit code so benchmark data is only meaningful + when the command succeeds (exit code 0). When ``--yes`` is not yet + implemented the benchmark measures error-path latency; after the + fix it measures real init latency. + """ + with patch( + "cleveragents.application.container.get_container" + ) as mock_container: + mock_service = create_autospec(ProjectService, instance=True) + mock_project = _MockProject() + mock_project.name = project_name + mock_project.path = Path(self._tmpdir) + # Pre-populate spec-required fields (specification.md:1381-1386) + # so benchmarks measure real output paths once #522 is fixed. + mock_project.data_dir = Path(self._tmpdir) + mock_project.config_path = Path(self._tmpdir) / "config.toml" + mock_project.database_status = "initialized (schema v3)" + mock_project.directories = ["logs", "cache", "sessions", "contexts"] + mock_service.initialize_project.return_value = mock_project + mock_container.return_value.project_service.return_value = mock_service + + result = self._runner.invoke(app, args) + self._last_exit_code = result.exit_code + + def time_init_yes_flag(self) -> None: + """Measure end-to-end latency of ``agents init --yes``.""" + self._invoke_with_mock(["init", "--yes"], "bench-project") + + def time_init_short_y_flag(self) -> None: + """Measure end-to-end latency of ``agents init -y``.""" + self._invoke_with_mock(["init", "-y"], "bench-project") + + def time_init_yes_flag_with_path(self) -> None: + """Measure latency of ``agents init --yes --path ``. + + NOTE: ``--path`` is an implementation detail of the current ``init`` + command. The spec (``specification.md:1217``) defines only + ``agents init [--yes|-y]`` with no ``--path`` option. When #522 + aligns the command with the spec this benchmark may need to be + removed or updated. + """ + self._invoke_with_mock( + ["init", "--yes", "--path", self._tmpdir], "bench-path-project" + ) + + def track_exit_code(self) -> int: + """Track the CLI exit code as an ASV metric. + + Returns 0 when ``--yes`` is correctly implemented; non-zero while + the flag is missing. This lets ASV detect regressions that + reintroduce ``NoSuchOption`` without silently reporting error-path + latency. + """ + self._invoke_with_mock(["init", "--yes"], "bench-project") + return self._last_exit_code + + +InitYesFlagSuite.track_exit_code.unit = "exit_code" diff --git a/features/cli_init_yes_flag.feature b/features/cli_init_yes_flag.feature new file mode 100644 index 000000000..bbbabc58c --- /dev/null +++ b/features/cli_init_yes_flag.feature @@ -0,0 +1,56 @@ +# These tests target bug #522 and are expected to fail until the fix is applied. +# +# NOTE FOR FIX AUTHOR (#522): +# Scenarios 1-4 will fail for TWO independent reasons: +# (a) The --yes / -y flag is not yet implemented (NoSuchOption). +# (b) The current init_command output format (project.py) does not match +# the spec-defined output at docs/specification.md:1381-1402. +# The fix must address both: add the --yes flag AND remodel the output to +# match the spec (Data Dir, Config, Database, Directories fields with the +# "Initialized (non-interactive)" status message). +Feature: CLI init --yes flag for non-interactive initialization + As a developer using CleverAgents in CI or scripts + I want to run "agents init --yes" for non-interactive initialization + So that I can skip interactive prompts and use sensible defaults + + @tdd @bug522 @wip + Scenario: agents init --yes completes without error + Given I have a temporary project directory for init + When I run agents init with the --yes flag + Then the init command should exit with code 0 + And the project service initialize_project should have been called + + @tdd @bug522 @wip + Scenario: --yes suppresses interactive prompts + Given I have a temporary project directory for init + When I run agents init with the --yes flag + Then the init command should exit with code 0 + And the init output should contain "Initialized (non-interactive)" + And no interactive prompt should have been presented + + @tdd @bug522 @wip + Scenario: -y short-form alias completes without error + Given I have a temporary project directory for init + When I run agents init with the -y flag + Then the init command should exit with code 0 + And the init output should contain "Initialized (non-interactive)" + And the project service initialize_project should have been called + + @tdd @bug522 @wip + Scenario: Output includes expected initialization summary + Given I have a temporary project directory for init + When I run agents init with the --yes flag + Then the init command should exit with code 0 + And the init output should contain "Data Dir:" + And the init output should contain "Config:" + And the init output should contain "Database:" + And the init output should contain "Directories:" + And the init output should contain "logs, cache, sessions, contexts" + And the init output should contain "Initialized" + + @tdd @bug522 + Scenario: Interactive mode without --yes presents a prompt + Given I have a temporary project directory for init + When I run agents init without the --yes flag + Then the init command should exit with code 0 + And the init output should indicate interactive mode diff --git a/features/steps/cli_init_yes_flag_steps.py b/features/steps/cli_init_yes_flag_steps.py new file mode 100644 index 000000000..2c77b4801 --- /dev/null +++ b/features/steps/cli_init_yes_flag_steps.py @@ -0,0 +1,210 @@ +"""Step definitions for CLI init --yes flag tests (bug #522). + +These tests are TDD-style: they assert the CORRECT expected behaviour of +``agents init --yes`` and are expected to FAIL until the bug fix is applied. +""" + +from __future__ import annotations + +import os +import re +import shutil +import tempfile +from pathlib import Path +from unittest.mock import create_autospec, patch + +from behave import given, then, when +from typer.testing import CliRunner + +from cleveragents.application.services.project_service import ProjectService +from cleveragents.cli.main import app + + +def _restore_cwd(context): + """Restore the original working directory, env var, and clean up.""" + os.chdir(context._init_original_cwd) + if context._init_original_home is None: + os.environ.pop("CLEVERAGENTS_HOME", None) + else: + os.environ["CLEVERAGENTS_HOME"] = context._init_original_home + shutil.rmtree(context.temp_dir, ignore_errors=True) + + +def _create_init_mocks(context): + """Create and configure mocked container, service, and project. + + Returns ``(patcher, mock_service)`` where *patcher* is the started + ``patch`` context manager. + + The service mock uses ``create_autospec(ProjectService)`` so that + attribute access and method calls are validated against the real + ``ProjectService`` interface. The project mock uses a typed + ``_MockProject`` class instead of ``MagicMock`` because the + spec-required output fields (``data_dir``, ``config_path``, + ``database_status``, ``directories``) do not yet exist on the legacy + ``Project`` model — they will be added when the #522 fix aligns the + model with ``docs/specification.md:1381-1386``. Using + ``create_autospec(Project)`` would reject those attribute assignments. + """ + mock_service = create_autospec(ProjectService, instance=True) + + class _MockProject: + """Lightweight stand-in for the Project model with spec fields.""" + + name: str + path: Path + data_dir: Path + config_path: Path + database_status: str + directories: list[str] + + mock_project = _MockProject() + mock_project.name = Path(context.temp_dir).name + mock_project.path = Path(context.temp_dir) + # Pre-populate spec-required fields (specification.md:1381-1386) so + # that when the #522 fix lands, output assertions fail for the right + # reason (real bugs) rather than MagicMock stringification artefacts. + mock_project.data_dir = Path(context.temp_dir) + mock_project.config_path = Path(context.temp_dir) / "config.toml" + mock_project.database_status = "initialized (schema v3)" + mock_project.directories = ["logs", "cache", "sessions", "contexts"] + mock_service.initialize_project.return_value = mock_project + + patcher = patch("cleveragents.application.container.get_container") + mock_container = patcher.start() + mock_container.return_value.project_service.return_value = mock_service + + return patcher, mock_service + + +@given("I have a temporary project directory for init") +def step_temp_project_directory(context): + """Create a temporary directory and store it on *context*.""" + context.temp_dir = tempfile.mkdtemp() + # Use step-private attribute names (prefixed with _init_) to avoid + # colliding with environment.py's context.original_cwd (str). + # Match the framework's type (str via os.getcwd()) for consistency. + context._init_original_cwd = os.getcwd() + context._init_original_home = os.environ.get("CLEVERAGENTS_HOME") + os.environ["CLEVERAGENTS_HOME"] = context.temp_dir + os.chdir(context.temp_dir) + context.add_cleanup(_restore_cwd, context) + + +def _run_init_with_flag(context, flag: str) -> None: + """Invoke ``agents init`` with the given flag via the Typer test runner.""" + runner = CliRunner() + + patcher, mock_service = _create_init_mocks(context) + try: + result = runner.invoke(app, ["init", flag]) + finally: + patcher.stop() + + context.init_yes_result = { + "exit_code": result.exit_code, + "output": result.output, + } + context.init_yes_raw_result = result + context.init_yes_mock_service = mock_service + + +@when("I run agents init with the --yes flag") +def step_run_init_yes(context): + """Invoke ``agents init --yes`` via the Typer test runner.""" + _run_init_with_flag(context, "--yes") + + +@when("I run agents init with the -y flag") +def step_run_init_short_yes(context): + """Invoke ``agents init -y`` via the Typer test runner.""" + _run_init_with_flag(context, "-y") + + +@then("the init command should exit with code {code:d}") +def step_init_exit_code(context, code): + """Assert the init command exited with the expected code.""" + actual = context.init_yes_result["exit_code"] + assert actual == code, ( + f"Expected exit code {code}, got {actual}. " + f"Output: {context.init_yes_result['output']}" + ) + + +@then('the init output should contain "{text}"') +def step_init_output_contains(context, text): + """Assert that the init command output contains *text*.""" + output = context.init_yes_result["output"] + assert text in output, f"Expected '{text}' in output:\n{output}" + + +@then("the project service initialize_project should have been called") +def step_initialize_project_called(context): + """Assert the mock project service's initialize_project was invoked.""" + context.init_yes_mock_service.initialize_project.assert_called_once() + + +@when("I run agents init without the --yes flag") +def step_run_init_no_yes(context): + """Invoke ``agents init`` without --yes (interactive mode).""" + runner = CliRunner() + + patcher, _mock_service = _create_init_mocks(context) + try: + result = runner.invoke(app, ["init"]) + finally: + patcher.stop() + + context.init_yes_result = { + "exit_code": result.exit_code, + "output": result.output, + } + context.init_yes_raw_result = result + + +@then("the init output should indicate interactive mode") +def step_output_indicates_interactive(context): + """Assert that without --yes the output does NOT contain the + non-interactive marker, indicating the command ran in interactive mode. + + This is the negative complement to the --yes scenarios: when the fix + lands, ``agents init`` (without --yes) should either present a prompt + or omit the ``Initialized (non-interactive)`` marker. + """ + output = context.init_yes_result["output"] + assert "Initialized (non-interactive)" not in output, ( + "Expected interactive mode but output contains " + f"'Initialized (non-interactive)':\n{output}" + ) + + +@then("no interactive prompt should have been presented") +def step_no_interactive_prompt(context): + """Assert that no interactive prompt was presented. + + Verifies that the command output does not contain common prompt-like + tokens, which would indicate the ``--yes`` flag failed to suppress + interactive prompts. + """ + output = context.init_yes_result["output"] + prompt_tokens = ( + "[Y/n]", + "[y/N]", + "Continue? ", + "Proceed? ", + "Confirm ", + "(yes/no)", + ) + for token in prompt_tokens: + assert token not in output, ( + f"Unexpected prompt token '{token}' found in output:\n{output}" + ) + # Regex catches "Enter project name:", "Enter path:", "Enter value?" + # without false-positiving on "Entered configuration" or "Enterprise". + # Non-greedy .*? stops at the first : or ? to avoid over-matching + # when the line contains colons in non-prompt contexts (e.g. "Config: /path"). + enter_prompt = re.search(r"Enter\s+\S+.*?[:?]", output) + assert enter_prompt is None, ( + f"Unexpected Enter-style prompt found in output: " + f"'{enter_prompt.group()}'\n{output}" + ) diff --git a/noxfile.py b/noxfile.py index ac014bdd9..8489678f6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -574,6 +574,8 @@ def integration_tests(session: nox.Session): "discovery", "--exclude", "code_blocks", + "--exclude", + "wip", *robot_args, "robot/", ) diff --git a/robot/cli_init_yes_flag.robot b/robot/cli_init_yes_flag.robot new file mode 100644 index 000000000..873c5b2b2 --- /dev/null +++ b/robot/cli_init_yes_flag.robot @@ -0,0 +1,52 @@ +*** Settings *** +Documentation Integration smoke test for agents init --yes (bug #522). +... These tests are TDD-style and expected to FAIL until the +... bug fix is applied. +Resource ${CURDIR}/common.resource +Library Process +Library OperatingSystem +Library String +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Test Cases *** +Init Yes Flag Exits Without Error + [Documentation] agents init --yes should complete with exit code 0 + [Tags] wip + ${tmpdir}= Evaluate __import__('tempfile').mkdtemp(prefix='init_yes_') + ${result}= Run Process ${PYTHON} -m cleveragents init --yes + ... timeout=60s cwd=${tmpdir} + Should Be Equal As Integers ${result.rc} 0 + ... msg=Expected exit code 0 but got ${result.rc}. stderr: ${result.stderr} + [Teardown] Remove Directory ${tmpdir} recursive=True + +Init Yes Flag Produces Summary Output + [Documentation] agents init --yes should produce the initialization summary + ... with all spec-required output fields. + [Tags] wip + ${tmpdir}= Evaluate __import__('tempfile').mkdtemp(prefix='init_yes_out_') + ${result}= Run Process ${PYTHON} -m cleveragents init --yes + ... timeout=60s cwd=${tmpdir} + Should Be Equal As Integers ${result.rc} 0 + ... msg=Expected exit code 0 but got ${result.rc}. stderr: ${result.stderr} + Should Contain ${result.stdout} Initialized (non-interactive) + ... msg=Output should contain non-interactive initialization message per spec + Should Contain ${result.stdout} Data Dir: + ... msg=Output should contain Data Dir field per spec + Should Contain ${result.stdout} Config: + ... msg=Output should contain Config field per spec + Should Contain ${result.stdout} Database: + ... msg=Output should contain Database field per spec + Should Contain ${result.stdout} Directories: + ... msg=Output should contain Directories field per spec + [Teardown] Remove Directory ${tmpdir} recursive=True + +Init Short Y Flag Exits Without Error + [Documentation] agents init -y should complete with exit code 0 (short-form alias) + [Tags] wip + ${tmpdir}= Evaluate __import__('tempfile').mkdtemp(prefix='init_y_') + ${result}= Run Process ${PYTHON} -m cleveragents init -y + ... timeout=60s cwd=${tmpdir} + Should Be Equal As Integers ${result.rc} 0 + ... msg=Expected exit code 0 but got ${result.rc}. stderr: ${result.stderr} + [Teardown] Remove Directory ${tmpdir} recursive=True -- 2.52.0