diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f4736b2e..a659d5d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ auto-discovered with bounded scan_depth. Updated `fs-directory` child types and auto-discovery. Updated `git-checkout` child types. Includes YAML configs, Behave BDD tests, Robot tests, and ASV benchmarks. (#330) +- Added TDD bug-capture tests for #932 (plan apply missing --yes flag). (#950) - Fixed `plan execute` CLI failing with "Plan is not in an executable state (current: strategize/queued)" after strategize completed successfully. Root cause: `_get_plan_executor()` created a second `PlanLifecycleService` diff --git a/features/steps/tdd_plan_apply_yes_flag_steps.py b/features/steps/tdd_plan_apply_yes_flag_steps.py new file mode 100644 index 000000000..f53221c16 --- /dev/null +++ b/features/steps/tdd_plan_apply_yes_flag_steps.py @@ -0,0 +1,91 @@ +"""Step definitions for TDD Bug #932 — plan apply missing --yes flag. + +These steps verify that the ``lifecycle-apply`` CLI command accepts the +``--yes`` / ``-y`` flag as required by the specification. The spec +mandates ``agents plan apply [--yes|-y] `` to skip the +confirmation prompt before applying plan changes, but the current +implementation does not recognise this flag. + +When the flag is missing, Typer (Click) rejects the option with +``"No such option: --yes"`` and exit code 2. The assertion that this +error is absent will **fail** — proving the bug. The +``@tdd_expected_fail`` tag on the feature inverts this failure to a +pass while the bug remains open. +""" + +from __future__ import annotations + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.cli.commands.plan import app as plan_app + + +@given("a plan CLI runner for the yes-flag test") +def step_plan_cli_runner(context: Context) -> None: + """Create a Typer CliRunner for the plan sub-app.""" + context.apply_yes_runner = CliRunner() + + +@when("I invoke lifecycle-apply with --yes flag") +def step_invoke_lifecycle_apply_yes(context: Context) -> None: + """Invoke ``plan lifecycle-apply --yes ``. + + We pass a dummy plan ID because we only care whether the ``--yes`` + flag is recognised by the CLI framework, not whether the plan + exists. If ``--yes`` is unknown, Typer exits with code 2 *before* + any business logic runs. + """ + context.apply_yes_result = context.apply_yes_runner.invoke( + plan_app, + ["lifecycle-apply", "--yes", "DUMMY_PLAN_ID"], + ) + + +@when("I invoke lifecycle-apply with -y flag") +def step_invoke_lifecycle_apply_y(context: Context) -> None: + """Invoke ``plan lifecycle-apply -y ``.""" + context.apply_yes_result = context.apply_yes_runner.invoke( + plan_app, + ["lifecycle-apply", "-y", "DUMMY_PLAN_ID"], + ) + + +@then("the lifecycle-apply --yes invocation should not report an unknown option") +def step_assert_yes_recognised(context: Context) -> None: + """Assert the output does NOT contain the Click/Typer unknown-option error. + + When ``--yes`` is not defined on the command, Click produces:: + + Error: No such option: --yes + + and returns exit code 2. If the flag is properly defined, the + command proceeds past option parsing (and may fail later due to + the dummy plan ID, but that is irrelevant — we only test flag + recognition). + """ + output = context.apply_yes_result.output + exit_code = context.apply_yes_result.exit_code + # Check both the text message and the Click/Typer usage-error exit code. + # Exit code 2 specifically means "unrecognised option" in Click. + assert exit_code != 2 and "No such option" not in output, ( + f"The --yes flag was not recognised by lifecycle-apply.\n" + f"Exit code: {exit_code}\n" + f"Output:\n{output}" + ) + + +@then("the lifecycle-apply -y invocation should not report an unknown option") +def step_assert_y_recognised(context: Context) -> None: + """Assert the output does NOT contain the Click/Typer unknown-option error. + + Same check as the ``--yes`` variant but for the ``-y`` short flag. + """ + output = context.apply_yes_result.output + exit_code = context.apply_yes_result.exit_code + assert exit_code != 2 and "No such option" not in output, ( + f"The -y flag was not recognised by lifecycle-apply.\n" + f"Exit code: {exit_code}\n" + f"Output:\n{output}" + ) diff --git a/features/tdd_plan_apply_yes_flag.feature b/features/tdd_plan_apply_yes_flag.feature new file mode 100644 index 000000000..66476e82f --- /dev/null +++ b/features/tdd_plan_apply_yes_flag.feature @@ -0,0 +1,22 @@ +@tdd_expected_fail @tdd_bug @tdd_bug_932 +Feature: TDD Bug #932 — plan apply missing --yes flag + As a developer + I want to verify that `agents plan lifecycle-apply` accepts the --yes + flag required by the specification + So that the bug is captured and will be caught by a regression test + + The specification mandates `agents plan apply [--yes|-y] ` + but the current `lifecycle-apply` implementation does not accept + `--yes` or `-y`. Other destructive CLI commands (`session delete`, + `project delete`, `plan correct`, `plan rollback`) correctly implement + the `--yes`/`-y` pattern to skip confirmation prompts. + + Scenario: lifecycle-apply recognises the --yes long flag + Given a plan CLI runner for the yes-flag test + When I invoke lifecycle-apply with --yes flag + Then the lifecycle-apply --yes invocation should not report an unknown option + + Scenario: lifecycle-apply recognises the -y short flag + Given a plan CLI runner for the yes-flag test + When I invoke lifecycle-apply with -y flag + Then the lifecycle-apply -y invocation should not report an unknown option diff --git a/noxfile.py b/noxfile.py index 982e7b0e5..bc9365c4f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1060,7 +1060,7 @@ def benchmark_regression(session: nox.Session): """Run Airspeed Velocity benchmarks regression test.""" session.install("-e", ".[tests]") config_path = "asv.conf.json" - asv_base_sha = os.environ.get("ASV_BASE_SHA") + asv_base_sha = os.environ.get("ASV_BASE_SHA", "master") session.run( "asv", "machine", diff --git a/robot/helper_tdd_plan_apply_yes_flag.py b/robot/helper_tdd_plan_apply_yes_flag.py new file mode 100644 index 000000000..61d1e7767 --- /dev/null +++ b/robot/helper_tdd_plan_apply_yes_flag.py @@ -0,0 +1,93 @@ +"""Helper script for tdd_plan_apply_yes_flag.robot smoke tests. + +Each subcommand exercises the real CLI path (no mocks) to reproduce bug #932. +The ``lifecycle-apply`` command is invoked with ``--yes`` or ``-y`` to verify +that the flag is recognised by the CLI framework. + +The helper reports the **real** outcome: it exits 0 and prints the sentinel +when the flag is accepted (bug is fixed), and exits 1 when the flag is +rejected (bug still present). The ``tdd_expected_fail_listener`` on the +Robot side handles pass/fail inversion while the bug remains open. +""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path +from typing import NoReturn + +# Ensure local source tree is importable. +_ROOT = Path(__file__).resolve().parents[1] +_SRC = str(_ROOT / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 + +runner = CliRunner() + + +def _fail(message: str) -> NoReturn: + """Print an error message to stderr and exit with code 1.""" + print(message, file=sys.stderr) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def check_yes_long() -> None: + """Invoke ``lifecycle-apply --yes DUMMY`` and verify the flag is accepted. + + Exits 0 with sentinel when ``--yes`` is recognised (bug fixed). + Exits 1 when Typer rejects the flag (bug still present). + """ + result = runner.invoke(plan_app, ["lifecycle-apply", "--yes", "DUMMY_PLAN_ID"]) + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"lifecycle-apply rejected --yes flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + print("tdd-plan-apply-yes-flag-long-ok") + + +def check_yes_short() -> None: + """Invoke ``lifecycle-apply -y DUMMY`` and verify the flag is accepted. + + Exits 0 with sentinel when ``-y`` is recognised (bug fixed). + Exits 1 when Typer rejects the flag (bug still present). + """ + result = runner.invoke(plan_app, ["lifecycle-apply", "-y", "DUMMY_PLAN_ID"]) + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"lifecycle-apply rejected -y flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + print("tdd-plan-apply-yes-flag-short-ok") + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], None]] = { + "check-yes-long": check_yes_long, + "check-yes-short": check_yes_short, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print( + f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>", + file=sys.stderr, + ) + sys.exit(1) + cmd = _COMMANDS[sys.argv[1]] + cmd() diff --git a/robot/tdd_plan_apply_yes_flag.robot b/robot/tdd_plan_apply_yes_flag.robot new file mode 100644 index 000000000..5a9ea83aa --- /dev/null +++ b/robot/tdd_plan_apply_yes_flag.robot @@ -0,0 +1,33 @@ +*** Settings *** +Documentation TDD Bug #932 — plan lifecycle-apply missing --yes flag +... Integration smoke tests verifying that the lifecycle-apply +... command accepts the --yes / -y flag required by the +... specification. The spec mandates +... ``agents plan apply [--yes|-y] `` to skip the +... confirmation prompt, but the current implementation does not +... recognise this flag. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_tdd_plan_apply_yes_flag.py + +*** Test Cases *** +TDD Plan Apply Yes Long Flag Via CLI + [Documentation] Verify that ``lifecycle-apply --yes`` is recognised + [Tags] tdd_expected_fail tdd_bug tdd_bug_932 + ${result}= Run Process ${PYTHON} ${HELPER} check-yes-long cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-plan-apply-yes-flag-long-ok + +TDD Plan Apply Yes Short Flag Via CLI + [Documentation] Verify that ``lifecycle-apply -y`` is recognised + [Tags] tdd_expected_fail tdd_bug tdd_bug_932 + ${result}= Run Process ${PYTHON} ${HELPER} check-yes-short cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-plan-apply-yes-flag-short-ok