From 52ee04055eabb6014a18fb2176ac8f8ba79c7b37 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Mon, 23 Mar 2026 08:24:03 +0000 Subject: [PATCH 1/2] =?UTF-8?q?test:=20add=20TDD=20bug-capture=20test=20fo?= =?UTF-8?q?r=20#1038=20=E2=80=94=20validation=20add=20--required=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Write 4 Behave scenarios that prove bug #1038 exists: the `agents validation add` CLI command does not accept `--required` or `--informational` flags, despite the specification (§22334) requiring them. Passing either flag triggers a Typer NoSuchOption error. Scenarios tagged @tdd_bug @tdd_bug_1038 @tdd_expected_fail so CI passes via result inversion. Once the fix for #1038 is implemented and removes the @tdd_expected_fail tag, the tests will run normally. Test scenarios: 1. --required flag is accepted by validation add 2. --informational flag is accepted by validation add 3. --required flag overrides YAML config mode (informational -> required) 4. --informational flag overrides YAML config mode (required -> informational) YAML configs use flat `mode:` keys matching `Validation.from_config()` implementation (not the nested `validation.mode` schema from the spec), with a comment documenting the discrepancy. Mock uses side_effect to return the actual Validation object passed by the CLI, preventing false positives from hard-coded mock return values. Robot integration test: N/A — purely unit-level CLI argument parsing issue, no integration behavior to exercise. ISSUES CLOSED: #1102 --- CHANGELOG.md | 6 + .../tdd_validation_add_required_flag_steps.py | 259 +++++++++++++++ .../tdd_validation_add_required_flag.feature | 63 ++++ robot/helper_tdd_validation_required_flag.py | 309 ++++++++++++++++++ robot/tdd_validation_required_flag.robot | 70 ++++ 5 files changed, 707 insertions(+) create mode 100644 features/steps/tdd_validation_add_required_flag_steps.py create mode 100644 features/tdd_validation_add_required_flag.feature create mode 100644 robot/helper_tdd_validation_required_flag.py create mode 100644 robot/tdd_validation_required_flag.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f7f3251..e192b76a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,6 +236,12 @@ incorrectly loses to plan-level fallback (level 4). Two regression guard scenarios verify existing correct behaviour (plan override vs project override, project override vs host default). (#1101) +- Added TDD bug-capture tests for bug #1038 — `agents validation add` + missing `--required`/`--informational` flags. Four Behave BDD scenarios + (`@tdd_bug @tdd_bug_1038 @tdd_expected_fail`) verify that the `add` + command accepts `--required` and `--informational` flags and that + `--required` overrides the YAML config mode. Tests use + `@tdd_expected_fail` until the bug fix is merged. (#1102) - Added BuiltinAdapter class and MCP automatic resource slot creation. BuiltinAdapter wraps register_file_tools/register_git_tools/register_subplan_tool into a unified adapter interface. McpAdapter.infer_resource_slots() analyzes diff --git a/features/steps/tdd_validation_add_required_flag_steps.py b/features/steps/tdd_validation_add_required_flag_steps.py new file mode 100644 index 000000000..f592aa53c --- /dev/null +++ b/features/steps/tdd_validation_add_required_flag_steps.py @@ -0,0 +1,259 @@ +"""Step definitions for TDD bug #1038. + +``agents validation add`` missing ``--required`` flag. + +This test captures bug #1038. The specification +(``docs/specification.md`` line 22334) states that the validation mode +can be set "via ``--required``/``--informational`` on ``agents validation +add``", and numerous workflow examples throughout the spec use +``--required``. However, the current implementation of the ``add`` +command in ``cleveragents.cli.commands.validation`` does not define +``--required`` or ``--informational`` options, so passing either flag +causes a ``NoSuchOption`` error at runtime. + +NOTE -- Spec Contradiction: + Rui Hu's investigation (issue #1038 comment #70755) found that the + formal CLI reference (specification.md lines 9279-9290) does NOT + include --required/--informational flags -- they appear only in + walkthrough examples and specification.md line 22334. Additionally, + specification.md line 30761 states: "For entity registration commands + (actor add, skill add, tool add, validation add, ...), the YAML + configuration file is the sole source of truth -- the --config file + fully defines the entity and no CLI override flags are accepted." + The resolution may be to add the flags to the CLI OR to clean up the + spec. See #1038. + +The ``@tdd_expected_fail`` tag on the scenarios inverts the result: these +tests *pass* CI because the underlying assertions *fail* (proving the bug +exists). Once the fix for #1038 is merged and the flags are implemented, +the ``@tdd_expected_fail`` tag must be removed so the tests run normally. +""" + +from __future__ import annotations + +import contextlib +import os +import tempfile +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.cli.commands.validation import app as validation_app + +_runner = CliRunner() + +_PATCH_SVC = "cleveragents.cli.commands.validation._get_tool_registry_service" + +_VALID_MODES = {"required", "informational"} + + +def _safe_unlink(path: str) -> None: + """Remove a file if it exists, silently ignoring missing files.""" + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a tdd 1038 CLI test runner with mocked services") +def step_tdd_1038_background(context: Context) -> None: + """Set up CliRunner and a mocked ToolRegistryService. + + The mock uses ``side_effect = lambda v: v`` so that ``register_tool`` + returns the actual Validation object the CLI passes in, rather than a + hard-coded mock. This ensures the CLI output reflects the real + Validation produced by the CLI's processing pipeline, not mock + configuration. + """ + context.tdd1038_runner = _runner + context.tdd1038_result = None + context.tdd1038_config_path = None + + mock_service = MagicMock() + + # Return whatever Validation object the CLI passes in, so the CLI + # output reflects actual CLI processing rather than mock configuration. + mock_service.register_tool.side_effect = lambda v: v + context.tdd1038_mock_service = mock_service + + +@given("a tdd 1038 temporary validation YAML config") +def step_tdd_1038_temp_config(context: Context) -> None: + """Create a temporary YAML config file with mode ``required``. + + The YAML uses a flat ``mode`` key to match what + ``Validation.from_config()`` actually reads (``config.get("mode", + "required")``). Note: the specification's Validation Configuration + schema (specification.md line 34109) nests mode under a ``validation`` + key, but the current ``from_config()`` implementation reads a flat + top-level ``mode`` key. This test uses the flat format so the config + is actually effective and the false-positive guard is meaningful. + """ + config_content = ( + "name: local/unit-tests\n" + "description: Unit tests\n" + "source: custom\n" + "mode: required\n" + "code: |\n" + " def run(inputs):\n" + " return {'passed': True}\n" + ) + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + delete=False, + ) as tmp: + tmp.write(config_content) + config_path = tmp.name + context.tdd1038_config_path = config_path + context.add_cleanup(lambda: _safe_unlink(config_path)) + + +@given('a tdd 1038 temporary validation YAML config with mode "{mode}"') +def step_tdd_1038_temp_config_with_mode(context: Context, mode: str) -> None: + """Create a temporary YAML config file with a specific mode. + + The YAML uses a flat ``mode`` key to match what + ``Validation.from_config()`` actually reads (``config.get("mode", + "required")``). See the docstring on ``step_tdd_1038_temp_config`` + for the rationale on flat vs nested format. + """ + assert mode in _VALID_MODES, f"Invalid mode '{mode}': must be one of {_VALID_MODES}" + config_content = ( + "name: local/unit-tests\n" + "description: Unit tests\n" + "source: custom\n" + f"mode: {mode}\n" + "code: |\n" + " def run(inputs):\n" + " return {'passed': True}\n" + ) + # Clean up any previous config from the Background step. + if context.tdd1038_config_path is not None: + _safe_unlink(context.tdd1038_config_path) + + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + delete=False, + ) as tmp: + tmp.write(config_content) + config_path = tmp.name + context.tdd1038_config_path = config_path + context.add_cleanup(lambda: _safe_unlink(config_path)) + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when("I tdd 1038 invoke validation add with --required flag") +def step_tdd_1038_add_required(context: Context) -> None: + """Invoke ``agents validation add --config --required``. + + Per the spec, the ``--required`` flag should set the validation mode to + ``required``, overriding whatever mode the YAML config defines. The + current implementation does NOT have this flag, so Typer raises + ``NoSuchOption`` -- which is the bug this test captures. + + Note: the positional NAME argument shown in the original bug report + (#1038) is omitted here because NAME handling is a separate spec + inconsistency not under test in this scenario. This test focuses + solely on the missing ``--required``/``--informational`` flags. + """ + with patch(_PATCH_SVC, return_value=context.tdd1038_mock_service): + context.tdd1038_result = context.tdd1038_runner.invoke( + validation_app, + [ + "add", + "--config", + context.tdd1038_config_path, + "--required", + "--format", + "plain", + ], + ) + + +@when("I tdd 1038 invoke validation add with --informational flag") +def step_tdd_1038_add_informational(context: Context) -> None: + """Invoke ``agents validation add --config --informational``. + + Per the spec (specification.md line 22334), ``--informational`` sets + the mode to ``informational``. This flag is also missing from the + current CLI. + """ + with patch(_PATCH_SVC, return_value=context.tdd1038_mock_service): + context.tdd1038_result = context.tdd1038_runner.invoke( + validation_app, + [ + "add", + "--config", + context.tdd1038_config_path, + "--informational", + "--format", + "plain", + ], + ) + + +# --------------------------------------------------------------------------- +# Then steps +# --------------------------------------------------------------------------- + + +@then("the tdd 1038 CLI result should succeed") +def step_tdd_1038_result_succeed(context: Context) -> None: + """Assert the CLI invocation completed successfully (exit code 0). + + This assertion FAILS while bug #1038 is present because ``--required`` + and ``--informational`` are not recognised options, causing a non-zero + exit. The ``@tdd_expected_fail`` tag inverts this failure into a pass. + """ + result = context.tdd1038_result + assert result is not None, "No CLI result captured" + assert result.exit_code == 0, ( + f"Expected exit code 0 but got {result.exit_code}. Output:\n{result.output}" + ) + + +@then('the tdd 1038 registered validation mode should be "{expected_mode}"') +def step_tdd_1038_mode_check(context: Context, expected_mode: str) -> None: + """Assert the output contains the expected mode and the service received it. + + When the bug is fixed, the CLI should accept the ``--required`` / + ``--informational`` flag and the rendered output should include the + mode accordingly. + + In addition to checking the CLI output, this step verifies that + ``register_tool`` was called with a Validation object whose ``mode`` + attribute matches the expected value. This prevents a false positive + where the mock always returns a hard-coded mode regardless of whether + the CLI actually forwarded the flag to the service layer. + """ + result = context.tdd1038_result + assert result is not None, "No CLI result captured" + assert f"mode: {expected_mode}" in result.output, ( + f"Expected 'mode: {expected_mode}' in output, got:\n{result.output}" + ) + + # Verify the service layer received the correct mode -- guards against + # false positives from hard-coded mock return values. + mock_service = context.tdd1038_mock_service + assert mock_service.register_tool.called, ( + "register_tool was never called on the mock service" + ) + call_args: Any = mock_service.register_tool.call_args + validation_arg = call_args[0][0] + actual_mode = getattr(validation_arg, "mode", None) + assert actual_mode == expected_mode, ( + f"Expected register_tool to receive mode '{expected_mode}', " + f"but the Validation object had mode '{actual_mode}'" + ) diff --git a/features/tdd_validation_add_required_flag.feature b/features/tdd_validation_add_required_flag.feature new file mode 100644 index 000000000..910963a82 --- /dev/null +++ b/features/tdd_validation_add_required_flag.feature @@ -0,0 +1,63 @@ +# TDD bug-capture test for bug #1038. +# +# The specification (docs/specification.md line 22334) states that the +# validation mode can be set "via --required/--informational on agents +# validation add", and numerous workflow examples in the spec use the +# --required flag. However, the current implementation of the ``add`` +# command in ``cleveragents.cli.commands.validation`` does not accept +# --required or --informational flags, causing a ``NoSuchOption`` error +# at runtime. +# +# NOTE — Spec Contradiction: +# Rui Hu's investigation (issue #1038 comment #70755) found that the +# formal CLI reference (specification.md lines 9279-9290) does NOT include +# --required/--informational flags — they appear only in walkthrough +# examples and specification.md line 22334. Additionally, +# specification.md line 30761 states: "For entity registration commands +# (actor add, skill add, tool add, validation add, …), the YAML +# configuration file is the sole source of truth — the --config file +# fully defines the entity and no CLI override flags are accepted." +# The resolution may be to add the flags to the CLI OR to clean up the +# spec. See #1038. +# +# These scenarios assert the CORRECT expected behavior. Because the bug is +# still present, the underlying assertions will fail — the @tdd_expected_fail +# tag inverts the result so the test suite passes CI. Once bug #1038 is fixed +# and the --required/--informational flags are implemented, the +# @tdd_expected_fail tag must be removed so the test runs normally. +# +# NOTE — Deferred edge case: mutual exclusivity of --required and +# --informational when both are passed simultaneously is not tested here. +# That edge case is deferred to the bug-fix PR for #1038. + +@tdd_bug @tdd_bug_1038 @tdd_expected_fail +Feature: Bug #1038 — validation add missing --required flag + As a user of the CleverAgents CLI + I want the ``agents validation add`` command to accept a ``--required`` flag + So that I can override the validation mode at registration time per the spec + + Background: + Given a tdd 1038 CLI test runner with mocked services + And a tdd 1038 temporary validation YAML config + + Scenario: The --required flag is accepted by validation add + When I tdd 1038 invoke validation add with --required flag + Then the tdd 1038 CLI result should succeed + And the tdd 1038 registered validation mode should be "required" + + Scenario: The --informational flag is accepted by validation add + When I tdd 1038 invoke validation add with --informational flag + Then the tdd 1038 CLI result should succeed + And the tdd 1038 registered validation mode should be "informational" + + Scenario: The --required flag overrides YAML config mode + Given a tdd 1038 temporary validation YAML config with mode "informational" + When I tdd 1038 invoke validation add with --required flag + Then the tdd 1038 CLI result should succeed + And the tdd 1038 registered validation mode should be "required" + + Scenario: The --informational flag overrides YAML config mode + Given a tdd 1038 temporary validation YAML config with mode "required" + When I tdd 1038 invoke validation add with --informational flag + Then the tdd 1038 CLI result should succeed + And the tdd 1038 registered validation mode should be "informational" diff --git a/robot/helper_tdd_validation_required_flag.py b/robot/helper_tdd_validation_required_flag.py new file mode 100644 index 000000000..4ad686586 --- /dev/null +++ b/robot/helper_tdd_validation_required_flag.py @@ -0,0 +1,309 @@ +"""Helper script for tdd_validation_required_flag.robot smoke tests. + +Each subcommand exercises the ``agents validation add`` CLI path via +``typer.testing.CliRunner`` to reproduce bug #1038. The specification +(``docs/specification.md`` line 22334) states that the validation mode can +be set "via ``--required``/``--informational`` on ``agents validation add``", +and numerous workflow examples in the spec use the ``--required`` flag. +However, the current implementation of the ``add`` command in +``cleveragents.cli.commands.validation`` does not define ``--required`` or +``--informational`` options, causing a ``NoSuchOption`` error at runtime. + +The helper reports the **real** outcome: it exits 0 and prints the sentinel +when the expected behaviour is observed (bug fixed), and exits 1 when the +bug is still present. The ``tdd_expected_fail_listener`` on the Robot side +handles pass/fail inversion while the bug remains open. + +NOTE -- Spec Contradiction: + Rui Hu's investigation (issue #1038 comment #70755) found that the + formal CLI reference (specification.md lines 9279-9290) does NOT include + --required/--informational flags -- they appear only in walkthrough + examples and specification.md line 22334. Additionally, + specification.md line 30761 states: "For entity registration commands + (actor add, skill add, tool add, validation add, ...), the YAML + configuration file is the sole source of truth -- the --config file + fully defines the entity and no CLI override flags are accepted." + The resolution may be to add the flags to the CLI OR to clean up the + spec. See #1038. + +This test was written to capture bug #1038 per ticket #1102. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import NoReturn +from unittest.mock import MagicMock, patch + +# Ensure local source tree is importable. +_ROOT: Path = Path(__file__).resolve().parents[1] +_SRC: str = str(_ROOT / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.validation import app as validation_app # noqa: E402 + +runner: CliRunner = CliRunner() + +_PATCH_SVC: str = "cleveragents.cli.commands.validation._get_tool_registry_service" + + +def _fail(message: str) -> NoReturn: + """Print an error message to stderr and exit with code 1.""" + print(message, file=sys.stderr) + sys.exit(1) + + +def _safe_unlink(path: str) -> None: + """Remove a file if it exists, silently ignoring missing files.""" + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + + +def _create_yaml_config(mode: str = "required") -> str: + """Create a temporary YAML config file with the given mode. + + Uses a flat ``mode`` key to match what ``Validation.from_config()`` + actually reads (``config.get("mode", "required")``). + """ + config_content: str = ( + "name: local/unit-tests\n" + "description: Unit tests\n" + "source: custom\n" + f"mode: {mode}\n" + "code: |\n" + " def run(inputs):\n" + " return {'passed': True}\n" + ) + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + delete=False, + ) as tmp: + tmp.write(config_content) + return tmp.name + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def _check_required() -> None: + """Invoke ``validation add --config --required`` and verify accepted. + + Exits 0 with sentinel when ``--required`` is recognised (bug fixed). + Exits 1 when Typer rejects the flag (bug still present). + """ + config_path: str = _create_yaml_config() + try: + mock_service: MagicMock = MagicMock() + mock_service.register_tool.side_effect = lambda v: v + + with patch(_PATCH_SVC, return_value=mock_service): + result = runner.invoke( + validation_app, + ["add", "--config", config_path, "--required", "--format", "plain"], + ) + + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"validation add rejected --required flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + if result.exit_code != 0: + _fail( + f"validation add --required exited with code {result.exit_code}.\n" + f"Output: {result.output}" + ) + print("tdd-validation-required-flag-ok") + finally: + _safe_unlink(config_path) + + +def _check_informational() -> None: + """Invoke ``validation add --config --informational`` and verify accepted. + + Exits 0 with sentinel when ``--informational`` is recognised (bug fixed). + Exits 1 when Typer rejects the flag (bug still present). + """ + config_path: str = _create_yaml_config() + try: + mock_service: MagicMock = MagicMock() + mock_service.register_tool.side_effect = lambda v: v + + with patch(_PATCH_SVC, return_value=mock_service): + result = runner.invoke( + validation_app, + [ + "add", + "--config", + config_path, + "--informational", + "--format", + "plain", + ], + ) + + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"validation add rejected --informational flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + if result.exit_code != 0: + _fail( + f"validation add --informational exited with code {result.exit_code}.\n" + f"Output: {result.output}" + ) + print("tdd-validation-informational-flag-ok") + finally: + _safe_unlink(config_path) + + +def _check_required_overrides_config() -> None: + """Invoke with ``--required`` on a YAML that has ``mode: informational``. + + Verifies that the ``--required`` CLI flag overrides the YAML config mode. + Exits 0 with sentinel when the override works (bug fixed). + Exits 1 when the flag is not recognised (bug still present). + """ + config_path: str = _create_yaml_config(mode="informational") + try: + mock_service: MagicMock = MagicMock() + mock_service.register_tool.side_effect = lambda v: v + + with patch(_PATCH_SVC, return_value=mock_service): + result = runner.invoke( + validation_app, + ["add", "--config", config_path, "--required", "--format", "plain"], + ) + + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"validation add rejected --required flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + if result.exit_code != 0: + _fail( + f"validation add --required exited with code {result.exit_code}.\n" + f"Output: {result.output}" + ) + + # Verify the mode was overridden to "required" + if "mode: required" not in result.output: + _fail( + f"--required flag did not override YAML config mode.\n" + f"Expected 'mode: required' in output.\n" + f"Output: {result.output}" + ) + + # Verify the service layer received the correct mode + if mock_service.register_tool.called: + call_args = mock_service.register_tool.call_args + validation_arg = call_args[0][0] + actual_mode = getattr(validation_arg, "mode", None) + if actual_mode != "required": + _fail( + f"register_tool received mode '{actual_mode}' instead of " + f"'required'. The --required flag did not override the " + f"YAML config mode at the service layer." + ) + + print("tdd-validation-required-overrides-config-ok") + finally: + _safe_unlink(config_path) + + +def _check_informational_overrides_config() -> None: + """Invoke with ``--informational`` on a YAML that has ``mode: required``. + + Verifies that the ``--informational`` CLI flag overrides the YAML config + mode. Exits 0 with sentinel when the override works (bug fixed). + Exits 1 when the flag is not recognised (bug still present). + """ + config_path: str = _create_yaml_config(mode="required") + try: + mock_service: MagicMock = MagicMock() + mock_service.register_tool.side_effect = lambda v: v + + with patch(_PATCH_SVC, return_value=mock_service): + result = runner.invoke( + validation_app, + [ + "add", + "--config", + config_path, + "--informational", + "--format", + "plain", + ], + ) + + if "No such option" in result.output or result.exit_code == 2: + _fail( + f"validation add rejected --informational flag.\n" + f"Exit code: {result.exit_code}\n" + f"Output: {result.output}" + ) + if result.exit_code != 0: + _fail( + f"validation add --informational exited with code " + f"{result.exit_code}.\n" + f"Output: {result.output}" + ) + + # Verify the mode was overridden to "informational" + if "mode: informational" not in result.output: + _fail( + f"--informational flag did not override YAML config mode.\n" + f"Expected 'mode: informational' in output.\n" + f"Output: {result.output}" + ) + + # Verify the service layer received the correct mode + if mock_service.register_tool.called: + call_args = mock_service.register_tool.call_args + validation_arg = call_args[0][0] + actual_mode = getattr(validation_arg, "mode", None) + if actual_mode != "informational": + _fail( + f"register_tool received mode '{actual_mode}' instead of " + f"'informational'. The --informational flag did not override " + f"the YAML config mode at the service layer." + ) + + print("tdd-validation-informational-overrides-config-ok") + finally: + _safe_unlink(config_path) + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], None]] = { + "check-required": _check_required, + "check-informational": _check_informational, + "check-required-overrides-config": _check_required_overrides_config, + "check-informational-overrides-config": _check_informational_overrides_config, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print( + f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>", + file=sys.stderr, + ) + sys.exit(1) + cmd: Callable[[], None] = _COMMANDS[sys.argv[1]] + cmd() diff --git a/robot/tdd_validation_required_flag.robot b/robot/tdd_validation_required_flag.robot new file mode 100644 index 000000000..f15e6b1a3 --- /dev/null +++ b/robot/tdd_validation_required_flag.robot @@ -0,0 +1,70 @@ +*** Settings *** +Documentation TDD Bug #1038 — validation add missing --required/--informational flags +... Integration smoke tests verifying that the ``agents validation add`` +... command accepts ``--required`` and ``--informational`` flags as described +... in the specification (specification.md line 22334). The current +... implementation does not define these flags, causing a ``NoSuchOption`` +... error at runtime. +... +... NOTE — Spec Contradiction: The formal CLI reference +... (specification.md lines 9279-9290) does NOT include these flags — they +... appear only in walkthrough examples and specification.md line 22334. +... Additionally, specification.md line 30761 states: "For entity registration +... commands (actor add, skill add, tool add, validation add, …), the YAML +... configuration file is the sole source of truth." The resolution may be +... to add the flags to the CLI OR to clean up the spec. See #1038. +... +... Tests are tagged tdd_expected_fail so CI passes via result inversion +... while the bug remains open. Once bug #1038 is fixed, the +... tdd_expected_fail tag must be removed. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_tdd_validation_required_flag.py + +*** Test Cases *** +TDD Validation Add Required Flag Accepted + [Documentation] Verify that ``validation add --config --required`` + ... is accepted by the CLI and sets the validation mode to + ... ``required``. + [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + ${result}= Run Process ${PYTHON} ${HELPER} check-required cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-validation-required-flag-ok + +TDD Validation Add Informational Flag Accepted + [Documentation] Verify that ``validation add --config --informational`` + ... is accepted by the CLI and sets the validation mode to + ... ``informational``. + [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + ${result}= Run Process ${PYTHON} ${HELPER} check-informational cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-validation-informational-flag-ok + +TDD Validation Add Required Flag Overrides YAML Config + [Documentation] Verify that ``--required`` overrides a YAML config that + ... specifies ``mode: informational``. Both CLI output and + ... the service layer should reflect ``mode: required``. + [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + ${result}= Run Process ${PYTHON} ${HELPER} check-required-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-validation-required-overrides-config-ok + +TDD Validation Add Informational Flag Overrides YAML Config + [Documentation] Verify that ``--informational`` overrides a YAML config + ... that specifies ``mode: required``. Both CLI output and + ... the service layer should reflect ``mode: informational``. + [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + ${result}= Run Process ${PYTHON} ${HELPER} check-informational-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-validation-informational-overrides-config-ok -- 2.52.0 From 150742524019b7715ce77d841c14a5ef1a25e130 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Sat, 28 Mar 2026 03:18:41 +0000 Subject: [PATCH 2/2] test(tdd): align #1038 expected-fail tags with schema --- features/tdd_validation_add_required_flag.feature | 2 +- robot/tdd_validation_required_flag.robot | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/features/tdd_validation_add_required_flag.feature b/features/tdd_validation_add_required_flag.feature index 910963a82..1856c09aa 100644 --- a/features/tdd_validation_add_required_flag.feature +++ b/features/tdd_validation_add_required_flag.feature @@ -30,7 +30,7 @@ # --informational when both are passed simultaneously is not tested here. # That edge case is deferred to the bug-fix PR for #1038. -@tdd_bug @tdd_bug_1038 @tdd_expected_fail +@tdd_expected_fail @tdd_issue @tdd_issue_1038 Feature: Bug #1038 — validation add missing --required flag As a user of the CleverAgents CLI I want the ``agents validation add`` command to accept a ``--required`` flag diff --git a/robot/tdd_validation_required_flag.robot b/robot/tdd_validation_required_flag.robot index f15e6b1a3..ec5ff2390 100644 --- a/robot/tdd_validation_required_flag.robot +++ b/robot/tdd_validation_required_flag.robot @@ -29,7 +29,7 @@ TDD Validation Add Required Flag Accepted [Documentation] Verify that ``validation add --config --required`` ... is accepted by the CLI and sets the validation mode to ... ``required``. - [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + [Tags] tdd_expected_fail tdd_issue tdd_issue_1038 ${result}= Run Process ${PYTHON} ${HELPER} check-required cwd=${WORKSPACE} timeout=30s on_timeout=kill Log ${result.stdout} Log ${result.stderr} @@ -40,7 +40,7 @@ TDD Validation Add Informational Flag Accepted [Documentation] Verify that ``validation add --config --informational`` ... is accepted by the CLI and sets the validation mode to ... ``informational``. - [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + [Tags] tdd_expected_fail tdd_issue tdd_issue_1038 ${result}= Run Process ${PYTHON} ${HELPER} check-informational cwd=${WORKSPACE} timeout=30s on_timeout=kill Log ${result.stdout} Log ${result.stderr} @@ -51,7 +51,7 @@ TDD Validation Add Required Flag Overrides YAML Config [Documentation] Verify that ``--required`` overrides a YAML config that ... specifies ``mode: informational``. Both CLI output and ... the service layer should reflect ``mode: required``. - [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + [Tags] tdd_expected_fail tdd_issue tdd_issue_1038 ${result}= Run Process ${PYTHON} ${HELPER} check-required-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill Log ${result.stdout} Log ${result.stderr} @@ -62,7 +62,7 @@ TDD Validation Add Informational Flag Overrides YAML Config [Documentation] Verify that ``--informational`` overrides a YAML config ... that specifies ``mode: required``. Both CLI output and ... the service layer should reflect ``mode: informational``. - [Tags] tdd_bug tdd_bug_1038 tdd_expected_fail + [Tags] tdd_expected_fail tdd_issue tdd_issue_1038 ${result}= Run Process ${PYTHON} ${HELPER} check-informational-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill Log ${result.stdout} Log ${result.stderr} -- 2.52.0