test: add TDD bug-capture test for #1038 — validation add --required flag #1133
@@ -236,6 +236,12 @@
|
||||
incorrectly loses to plan-level fallback (level 4). Two regression guard
|
||||
scenarios verify existing correct behaviour (plan override vs project
|
||||
override, project override vs host default). (#1101)
|
||||
- Added TDD bug-capture tests for bug #1038 — `agents validation add`
|
||||
missing `--required`/`--informational` flags. Four Behave BDD scenarios
|
||||
(`@tdd_bug @tdd_bug_1038 @tdd_expected_fail`) verify that the `add`
|
||||
command accepts `--required` and `--informational` flags and that
|
||||
`--required` overrides the YAML config mode. Tests use
|
||||
`@tdd_expected_fail` until the bug fix is merged. (#1102)
|
||||
- Added BuiltinAdapter class and MCP automatic resource slot creation.
|
||||
BuiltinAdapter wraps register_file_tools/register_git_tools/register_subplan_tool
|
||||
into a unified adapter interface. McpAdapter.infer_resource_slots() analyzes
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Step definitions for TDD bug #1038.
|
||||
|
||||
``agents validation add`` missing ``--required`` flag.
|
||||
|
||||
This test captures bug #1038. The specification
|
||||
(``docs/specification.md`` line 22334) states that the validation mode
|
||||
can be set "via ``--required``/``--informational`` on ``agents validation
|
||||
add``", and numerous workflow examples throughout the spec use
|
||||
``--required``. However, the current implementation of the ``add``
|
||||
command in ``cleveragents.cli.commands.validation`` does not define
|
||||
``--required`` or ``--informational`` options, so passing either flag
|
||||
causes a ``NoSuchOption`` error at runtime.
|
||||
|
||||
NOTE -- Spec Contradiction:
|
||||
Rui Hu's investigation (issue #1038 comment #70755) found that the
|
||||
formal CLI reference (specification.md lines 9279-9290) does NOT
|
||||
include --required/--informational flags -- they appear only in
|
||||
walkthrough examples and specification.md line 22334. Additionally,
|
||||
specification.md line 30761 states: "For entity registration commands
|
||||
(actor add, skill add, tool add, validation add, ...), the YAML
|
||||
configuration file is the sole source of truth -- the --config file
|
||||
fully defines the entity and no CLI override flags are accepted."
|
||||
The resolution may be to add the flags to the CLI OR to clean up the
|
||||
spec. See #1038.
|
||||
|
||||
The ``@tdd_expected_fail`` tag on the scenarios inverts the result: these
|
||||
tests *pass* CI because the underlying assertions *fail* (proving the bug
|
||||
exists). Once the fix for #1038 is merged and the flags are implemented,
|
||||
the ``@tdd_expected_fail`` tag must be removed so the tests run normally.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands.validation import app as validation_app
|
||||
|
||||
_runner = CliRunner()
|
||||
|
||||
_PATCH_SVC = "cleveragents.cli.commands.validation._get_tool_registry_service"
|
||||
|
||||
_VALID_MODES = {"required", "informational"}
|
||||
|
||||
|
||||
def _safe_unlink(path: str) -> None:
|
||||
"""Remove a file if it exists, silently ignoring missing files."""
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a tdd 1038 CLI test runner with mocked services")
|
||||
def step_tdd_1038_background(context: Context) -> None:
|
||||
"""Set up CliRunner and a mocked ToolRegistryService.
|
||||
|
||||
The mock uses ``side_effect = lambda v: v`` so that ``register_tool``
|
||||
returns the actual Validation object the CLI passes in, rather than a
|
||||
hard-coded mock. This ensures the CLI output reflects the real
|
||||
Validation produced by the CLI's processing pipeline, not mock
|
||||
configuration.
|
||||
"""
|
||||
context.tdd1038_runner = _runner
|
||||
context.tdd1038_result = None
|
||||
context.tdd1038_config_path = None
|
||||
|
||||
mock_service = MagicMock()
|
||||
|
||||
# Return whatever Validation object the CLI passes in, so the CLI
|
||||
# output reflects actual CLI processing rather than mock configuration.
|
||||
mock_service.register_tool.side_effect = lambda v: v
|
||||
context.tdd1038_mock_service = mock_service
|
||||
|
||||
|
||||
@given("a tdd 1038 temporary validation YAML config")
|
||||
def step_tdd_1038_temp_config(context: Context) -> None:
|
||||
"""Create a temporary YAML config file with mode ``required``.
|
||||
|
||||
The YAML uses a flat ``mode`` key to match what
|
||||
``Validation.from_config()`` actually reads (``config.get("mode",
|
||||
"required")``). Note: the specification's Validation Configuration
|
||||
schema (specification.md line 34109) nests mode under a ``validation``
|
||||
key, but the current ``from_config()`` implementation reads a flat
|
||||
top-level ``mode`` key. This test uses the flat format so the config
|
||||
is actually effective and the false-positive guard is meaningful.
|
||||
"""
|
||||
config_content = (
|
||||
"name: local/unit-tests\n"
|
||||
"description: Unit tests\n"
|
||||
"source: custom\n"
|
||||
"mode: required\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': True}\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
config_path = tmp.name
|
||||
context.tdd1038_config_path = config_path
|
||||
context.add_cleanup(lambda: _safe_unlink(config_path))
|
||||
|
||||
|
||||
@given('a tdd 1038 temporary validation YAML config with mode "{mode}"')
|
||||
def step_tdd_1038_temp_config_with_mode(context: Context, mode: str) -> None:
|
||||
"""Create a temporary YAML config file with a specific mode.
|
||||
|
||||
The YAML uses a flat ``mode`` key to match what
|
||||
``Validation.from_config()`` actually reads (``config.get("mode",
|
||||
"required")``). See the docstring on ``step_tdd_1038_temp_config``
|
||||
for the rationale on flat vs nested format.
|
||||
"""
|
||||
assert mode in _VALID_MODES, f"Invalid mode '{mode}': must be one of {_VALID_MODES}"
|
||||
config_content = (
|
||||
"name: local/unit-tests\n"
|
||||
"description: Unit tests\n"
|
||||
"source: custom\n"
|
||||
f"mode: {mode}\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': True}\n"
|
||||
)
|
||||
# Clean up any previous config from the Background step.
|
||||
if context.tdd1038_config_path is not None:
|
||||
_safe_unlink(context.tdd1038_config_path)
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
config_path = tmp.name
|
||||
context.tdd1038_config_path = config_path
|
||||
context.add_cleanup(lambda: _safe_unlink(config_path))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I tdd 1038 invoke validation add with --required flag")
|
||||
def step_tdd_1038_add_required(context: Context) -> None:
|
||||
"""Invoke ``agents validation add --config <file> --required``.
|
||||
|
||||
Per the spec, the ``--required`` flag should set the validation mode to
|
||||
``required``, overriding whatever mode the YAML config defines. The
|
||||
current implementation does NOT have this flag, so Typer raises
|
||||
``NoSuchOption`` -- which is the bug this test captures.
|
||||
|
||||
Note: the positional NAME argument shown in the original bug report
|
||||
(#1038) is omitted here because NAME handling is a separate spec
|
||||
inconsistency not under test in this scenario. This test focuses
|
||||
solely on the missing ``--required``/``--informational`` flags.
|
||||
"""
|
||||
with patch(_PATCH_SVC, return_value=context.tdd1038_mock_service):
|
||||
context.tdd1038_result = context.tdd1038_runner.invoke(
|
||||
validation_app,
|
||||
[
|
||||
"add",
|
||||
"--config",
|
||||
context.tdd1038_config_path,
|
||||
"--required",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@when("I tdd 1038 invoke validation add with --informational flag")
|
||||
def step_tdd_1038_add_informational(context: Context) -> None:
|
||||
"""Invoke ``agents validation add --config <file> --informational``.
|
||||
|
||||
Per the spec (specification.md line 22334), ``--informational`` sets
|
||||
the mode to ``informational``. This flag is also missing from the
|
||||
current CLI.
|
||||
"""
|
||||
with patch(_PATCH_SVC, return_value=context.tdd1038_mock_service):
|
||||
context.tdd1038_result = context.tdd1038_runner.invoke(
|
||||
validation_app,
|
||||
[
|
||||
"add",
|
||||
"--config",
|
||||
context.tdd1038_config_path,
|
||||
"--informational",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the tdd 1038 CLI result should succeed")
|
||||
def step_tdd_1038_result_succeed(context: Context) -> None:
|
||||
"""Assert the CLI invocation completed successfully (exit code 0).
|
||||
|
||||
This assertion FAILS while bug #1038 is present because ``--required``
|
||||
and ``--informational`` are not recognised options, causing a non-zero
|
||||
exit. The ``@tdd_expected_fail`` tag inverts this failure into a pass.
|
||||
"""
|
||||
result = context.tdd1038_result
|
||||
assert result is not None, "No CLI result captured"
|
||||
assert result.exit_code == 0, (
|
||||
f"Expected exit code 0 but got {result.exit_code}. Output:\n{result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the tdd 1038 registered validation mode should be "{expected_mode}"')
|
||||
def step_tdd_1038_mode_check(context: Context, expected_mode: str) -> None:
|
||||
"""Assert the output contains the expected mode and the service received it.
|
||||
|
||||
When the bug is fixed, the CLI should accept the ``--required`` /
|
||||
``--informational`` flag and the rendered output should include the
|
||||
mode accordingly.
|
||||
|
||||
In addition to checking the CLI output, this step verifies that
|
||||
``register_tool`` was called with a Validation object whose ``mode``
|
||||
attribute matches the expected value. This prevents a false positive
|
||||
where the mock always returns a hard-coded mode regardless of whether
|
||||
the CLI actually forwarded the flag to the service layer.
|
||||
"""
|
||||
result = context.tdd1038_result
|
||||
assert result is not None, "No CLI result captured"
|
||||
assert f"mode: {expected_mode}" in result.output, (
|
||||
f"Expected 'mode: {expected_mode}' in output, got:\n{result.output}"
|
||||
)
|
||||
|
||||
# Verify the service layer received the correct mode -- guards against
|
||||
# false positives from hard-coded mock return values.
|
||||
mock_service = context.tdd1038_mock_service
|
||||
assert mock_service.register_tool.called, (
|
||||
"register_tool was never called on the mock service"
|
||||
)
|
||||
call_args: Any = mock_service.register_tool.call_args
|
||||
validation_arg = call_args[0][0]
|
||||
actual_mode = getattr(validation_arg, "mode", None)
|
||||
assert actual_mode == expected_mode, (
|
||||
f"Expected register_tool to receive mode '{expected_mode}', "
|
||||
f"but the Validation object had mode '{actual_mode}'"
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
# TDD bug-capture test for bug #1038.
|
||||
#
|
||||
# The specification (docs/specification.md line 22334) states that the
|
||||
# validation mode can be set "via --required/--informational on agents
|
||||
# validation add", and numerous workflow examples in the spec use the
|
||||
# --required flag. However, the current implementation of the ``add``
|
||||
# command in ``cleveragents.cli.commands.validation`` does not accept
|
||||
# --required or --informational flags, causing a ``NoSuchOption`` error
|
||||
# at runtime.
|
||||
#
|
||||
# NOTE — Spec Contradiction:
|
||||
# Rui Hu's investigation (issue #1038 comment #70755) found that the
|
||||
# formal CLI reference (specification.md lines 9279-9290) does NOT include
|
||||
# --required/--informational flags — they appear only in walkthrough
|
||||
# examples and specification.md line 22334. Additionally,
|
||||
# specification.md line 30761 states: "For entity registration commands
|
||||
# (actor add, skill add, tool add, validation add, …), the YAML
|
||||
# configuration file is the sole source of truth — the --config file
|
||||
# fully defines the entity and no CLI override flags are accepted."
|
||||
# The resolution may be to add the flags to the CLI OR to clean up the
|
||||
# spec. See #1038.
|
||||
#
|
||||
# These scenarios assert the CORRECT expected behavior. Because the bug is
|
||||
# still present, the underlying assertions will fail — the @tdd_expected_fail
|
||||
# tag inverts the result so the test suite passes CI. Once bug #1038 is fixed
|
||||
# and the --required/--informational flags are implemented, the
|
||||
# @tdd_expected_fail tag must be removed so the test runs normally.
|
||||
#
|
||||
# NOTE — Deferred edge case: mutual exclusivity of --required and
|
||||
# --informational when both are passed simultaneously is not tested here.
|
||||
# That edge case is deferred to the bug-fix PR for #1038.
|
||||
|
||||
@tdd_expected_fail @tdd_issue @tdd_issue_1038
|
||||
Feature: Bug #1038 — validation add missing --required flag
|
||||
As a user of the CleverAgents CLI
|
||||
I want the ``agents validation add`` command to accept a ``--required`` flag
|
||||
So that I can override the validation mode at registration time per the spec
|
||||
|
||||
Background:
|
||||
Given a tdd 1038 CLI test runner with mocked services
|
||||
And a tdd 1038 temporary validation YAML config
|
||||
|
||||
Scenario: The --required flag is accepted by validation add
|
||||
When I tdd 1038 invoke validation add with --required flag
|
||||
Then the tdd 1038 CLI result should succeed
|
||||
And the tdd 1038 registered validation mode should be "required"
|
||||
|
||||
Scenario: The --informational flag is accepted by validation add
|
||||
When I tdd 1038 invoke validation add with --informational flag
|
||||
Then the tdd 1038 CLI result should succeed
|
||||
And the tdd 1038 registered validation mode should be "informational"
|
||||
|
||||
Scenario: The --required flag overrides YAML config mode
|
||||
Given a tdd 1038 temporary validation YAML config with mode "informational"
|
||||
When I tdd 1038 invoke validation add with --required flag
|
||||
Then the tdd 1038 CLI result should succeed
|
||||
And the tdd 1038 registered validation mode should be "required"
|
||||
|
||||
Scenario: The --informational flag overrides YAML config mode
|
||||
Given a tdd 1038 temporary validation YAML config with mode "required"
|
||||
When I tdd 1038 invoke validation add with --informational flag
|
||||
Then the tdd 1038 CLI result should succeed
|
||||
And the tdd 1038 registered validation mode should be "informational"
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Helper script for tdd_validation_required_flag.robot smoke tests.
|
||||
|
||||
Each subcommand exercises the ``agents validation add`` CLI path via
|
||||
``typer.testing.CliRunner`` to reproduce bug #1038. The specification
|
||||
(``docs/specification.md`` line 22334) states that the validation mode can
|
||||
be set "via ``--required``/``--informational`` on ``agents validation add``",
|
||||
and numerous workflow examples in the spec use the ``--required`` flag.
|
||||
However, the current implementation of the ``add`` command in
|
||||
``cleveragents.cli.commands.validation`` does not define ``--required`` or
|
||||
``--informational`` options, causing a ``NoSuchOption`` error at runtime.
|
||||
|
||||
The helper reports the **real** outcome: it exits 0 and prints the sentinel
|
||||
when the expected behaviour is observed (bug fixed), and exits 1 when the
|
||||
bug is still present. The ``tdd_expected_fail_listener`` on the Robot side
|
||||
handles pass/fail inversion while the bug remains open.
|
||||
|
||||
NOTE -- Spec Contradiction:
|
||||
Rui Hu's investigation (issue #1038 comment #70755) found that the
|
||||
formal CLI reference (specification.md lines 9279-9290) does NOT include
|
||||
--required/--informational flags -- they appear only in walkthrough
|
||||
examples and specification.md line 22334. Additionally,
|
||||
specification.md line 30761 states: "For entity registration commands
|
||||
(actor add, skill add, tool add, validation add, ...), the YAML
|
||||
configuration file is the sole source of truth -- the --config file
|
||||
fully defines the entity and no CLI override flags are accepted."
|
||||
The resolution may be to add the flags to the CLI OR to clean up the
|
||||
spec. See #1038.
|
||||
|
||||
This test was written to capture bug #1038 per ticket #1102.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import NoReturn
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure local source tree is importable.
|
||||
_ROOT: Path = Path(__file__).resolve().parents[1]
|
||||
_SRC: str = str(_ROOT / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
|
||||
|
||||
runner: CliRunner = CliRunner()
|
||||
|
||||
_PATCH_SVC: str = "cleveragents.cli.commands.validation._get_tool_registry_service"
|
||||
|
||||
|
||||
def _fail(message: str) -> NoReturn:
|
||||
"""Print an error message to stderr and exit with code 1."""
|
||||
print(message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _safe_unlink(path: str) -> None:
|
||||
"""Remove a file if it exists, silently ignoring missing files."""
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def _create_yaml_config(mode: str = "required") -> str:
|
||||
"""Create a temporary YAML config file with the given mode.
|
||||
|
||||
Uses a flat ``mode`` key to match what ``Validation.from_config()``
|
||||
actually reads (``config.get("mode", "required")``).
|
||||
"""
|
||||
config_content: str = (
|
||||
"name: local/unit-tests\n"
|
||||
"description: Unit tests\n"
|
||||
"source: custom\n"
|
||||
f"mode: {mode}\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': True}\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
return tmp.name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _check_required() -> None:
|
||||
"""Invoke ``validation add --config <file> --required`` and verify accepted.
|
||||
|
||||
Exits 0 with sentinel when ``--required`` is recognised (bug fixed).
|
||||
Exits 1 when Typer rejects the flag (bug still present).
|
||||
"""
|
||||
config_path: str = _create_yaml_config()
|
||||
try:
|
||||
mock_service: MagicMock = MagicMock()
|
||||
mock_service.register_tool.side_effect = lambda v: v
|
||||
|
||||
with patch(_PATCH_SVC, return_value=mock_service):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
["add", "--config", config_path, "--required", "--format", "plain"],
|
||||
)
|
||||
|
||||
if "No such option" in result.output or result.exit_code == 2:
|
||||
_fail(
|
||||
f"validation add rejected --required flag.\n"
|
||||
f"Exit code: {result.exit_code}\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(
|
||||
f"validation add --required exited with code {result.exit_code}.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
print("tdd-validation-required-flag-ok")
|
||||
finally:
|
||||
_safe_unlink(config_path)
|
||||
|
||||
|
||||
def _check_informational() -> None:
|
||||
"""Invoke ``validation add --config <file> --informational`` and verify accepted.
|
||||
|
||||
Exits 0 with sentinel when ``--informational`` is recognised (bug fixed).
|
||||
Exits 1 when Typer rejects the flag (bug still present).
|
||||
"""
|
||||
config_path: str = _create_yaml_config()
|
||||
try:
|
||||
mock_service: MagicMock = MagicMock()
|
||||
mock_service.register_tool.side_effect = lambda v: v
|
||||
|
||||
with patch(_PATCH_SVC, return_value=mock_service):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
[
|
||||
"add",
|
||||
"--config",
|
||||
config_path,
|
||||
"--informational",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
if "No such option" in result.output or result.exit_code == 2:
|
||||
_fail(
|
||||
f"validation add rejected --informational flag.\n"
|
||||
f"Exit code: {result.exit_code}\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(
|
||||
f"validation add --informational exited with code {result.exit_code}.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
print("tdd-validation-informational-flag-ok")
|
||||
finally:
|
||||
_safe_unlink(config_path)
|
||||
|
||||
|
||||
def _check_required_overrides_config() -> None:
|
||||
"""Invoke with ``--required`` on a YAML that has ``mode: informational``.
|
||||
|
||||
Verifies that the ``--required`` CLI flag overrides the YAML config mode.
|
||||
Exits 0 with sentinel when the override works (bug fixed).
|
||||
Exits 1 when the flag is not recognised (bug still present).
|
||||
"""
|
||||
config_path: str = _create_yaml_config(mode="informational")
|
||||
try:
|
||||
mock_service: MagicMock = MagicMock()
|
||||
mock_service.register_tool.side_effect = lambda v: v
|
||||
|
||||
with patch(_PATCH_SVC, return_value=mock_service):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
["add", "--config", config_path, "--required", "--format", "plain"],
|
||||
)
|
||||
|
||||
if "No such option" in result.output or result.exit_code == 2:
|
||||
_fail(
|
||||
f"validation add rejected --required flag.\n"
|
||||
f"Exit code: {result.exit_code}\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(
|
||||
f"validation add --required exited with code {result.exit_code}.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
|
||||
# Verify the mode was overridden to "required"
|
||||
if "mode: required" not in result.output:
|
||||
_fail(
|
||||
f"--required flag did not override YAML config mode.\n"
|
||||
f"Expected 'mode: required' in output.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
|
||||
# Verify the service layer received the correct mode
|
||||
if mock_service.register_tool.called:
|
||||
call_args = mock_service.register_tool.call_args
|
||||
validation_arg = call_args[0][0]
|
||||
actual_mode = getattr(validation_arg, "mode", None)
|
||||
if actual_mode != "required":
|
||||
_fail(
|
||||
f"register_tool received mode '{actual_mode}' instead of "
|
||||
f"'required'. The --required flag did not override the "
|
||||
f"YAML config mode at the service layer."
|
||||
)
|
||||
|
||||
print("tdd-validation-required-overrides-config-ok")
|
||||
finally:
|
||||
_safe_unlink(config_path)
|
||||
|
||||
|
||||
def _check_informational_overrides_config() -> None:
|
||||
"""Invoke with ``--informational`` on a YAML that has ``mode: required``.
|
||||
|
||||
Verifies that the ``--informational`` CLI flag overrides the YAML config
|
||||
mode. Exits 0 with sentinel when the override works (bug fixed).
|
||||
Exits 1 when the flag is not recognised (bug still present).
|
||||
"""
|
||||
config_path: str = _create_yaml_config(mode="required")
|
||||
try:
|
||||
mock_service: MagicMock = MagicMock()
|
||||
mock_service.register_tool.side_effect = lambda v: v
|
||||
|
||||
with patch(_PATCH_SVC, return_value=mock_service):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
[
|
||||
"add",
|
||||
"--config",
|
||||
config_path,
|
||||
"--informational",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
if "No such option" in result.output or result.exit_code == 2:
|
||||
_fail(
|
||||
f"validation add rejected --informational flag.\n"
|
||||
f"Exit code: {result.exit_code}\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(
|
||||
f"validation add --informational exited with code "
|
||||
f"{result.exit_code}.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
|
||||
# Verify the mode was overridden to "informational"
|
||||
if "mode: informational" not in result.output:
|
||||
_fail(
|
||||
f"--informational flag did not override YAML config mode.\n"
|
||||
f"Expected 'mode: informational' in output.\n"
|
||||
f"Output: {result.output}"
|
||||
)
|
||||
|
||||
# Verify the service layer received the correct mode
|
||||
if mock_service.register_tool.called:
|
||||
call_args = mock_service.register_tool.call_args
|
||||
validation_arg = call_args[0][0]
|
||||
actual_mode = getattr(validation_arg, "mode", None)
|
||||
if actual_mode != "informational":
|
||||
_fail(
|
||||
f"register_tool received mode '{actual_mode}' instead of "
|
||||
f"'informational'. The --informational flag did not override "
|
||||
f"the YAML config mode at the service layer."
|
||||
)
|
||||
|
||||
print("tdd-validation-informational-overrides-config-ok")
|
||||
finally:
|
||||
_safe_unlink(config_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, Callable[[], None]] = {
|
||||
"check-required": _check_required,
|
||||
"check-informational": _check_informational,
|
||||
"check-required-overrides-config": _check_required_overrides_config,
|
||||
"check-informational-overrides-config": _check_informational_overrides_config,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(
|
||||
f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
cmd: Callable[[], None] = _COMMANDS[sys.argv[1]]
|
||||
cmd()
|
||||
@@ -0,0 +1,70 @@
|
||||
*** Settings ***
|
||||
Documentation TDD Bug #1038 — validation add missing --required/--informational flags
|
||||
... Integration smoke tests verifying that the ``agents validation add``
|
||||
... command accepts ``--required`` and ``--informational`` flags as described
|
||||
... in the specification (specification.md line 22334). The current
|
||||
... implementation does not define these flags, causing a ``NoSuchOption``
|
||||
... error at runtime.
|
||||
...
|
||||
... NOTE — Spec Contradiction: The formal CLI reference
|
||||
... (specification.md lines 9279-9290) does NOT include these flags — they
|
||||
... appear only in walkthrough examples and specification.md line 22334.
|
||||
... Additionally, specification.md line 30761 states: "For entity registration
|
||||
... commands (actor add, skill add, tool add, validation add, …), the YAML
|
||||
... configuration file is the sole source of truth." The resolution may be
|
||||
... to add the flags to the CLI OR to clean up the spec. See #1038.
|
||||
...
|
||||
... Tests are tagged tdd_expected_fail so CI passes via result inversion
|
||||
... while the bug remains open. Once bug #1038 is fixed, the
|
||||
... tdd_expected_fail tag must be removed.
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_tdd_validation_required_flag.py
|
||||
|
||||
*** Test Cases ***
|
||||
TDD Validation Add Required Flag Accepted
|
||||
[Documentation] Verify that ``validation add --config <file> --required``
|
||||
... is accepted by the CLI and sets the validation mode to
|
||||
... ``required``.
|
||||
[Tags] tdd_expected_fail tdd_issue tdd_issue_1038
|
||||
${result}= Run Process ${PYTHON} ${HELPER} check-required cwd=${WORKSPACE} timeout=30s on_timeout=kill
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} tdd-validation-required-flag-ok
|
||||
|
||||
TDD Validation Add Informational Flag Accepted
|
||||
[Documentation] Verify that ``validation add --config <file> --informational``
|
||||
... is accepted by the CLI and sets the validation mode to
|
||||
... ``informational``.
|
||||
[Tags] tdd_expected_fail tdd_issue tdd_issue_1038
|
||||
${result}= Run Process ${PYTHON} ${HELPER} check-informational cwd=${WORKSPACE} timeout=30s on_timeout=kill
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} tdd-validation-informational-flag-ok
|
||||
|
||||
TDD Validation Add Required Flag Overrides YAML Config
|
||||
[Documentation] Verify that ``--required`` overrides a YAML config that
|
||||
... specifies ``mode: informational``. Both CLI output and
|
||||
... the service layer should reflect ``mode: required``.
|
||||
[Tags] tdd_expected_fail tdd_issue tdd_issue_1038
|
||||
${result}= Run Process ${PYTHON} ${HELPER} check-required-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} tdd-validation-required-overrides-config-ok
|
||||
|
||||
TDD Validation Add Informational Flag Overrides YAML Config
|
||||
[Documentation] Verify that ``--informational`` overrides a YAML config
|
||||
... that specifies ``mode: required``. Both CLI output and
|
||||
... the service layer should reflect ``mode: informational``.
|
||||
[Tags] tdd_expected_fail tdd_issue tdd_issue_1038
|
||||
${result}= Run Process ${PYTHON} ${HELPER} check-informational-overrides-config cwd=${WORKSPACE} timeout=30s on_timeout=kill
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} tdd-validation-informational-overrides-config-ok
|
||||
Reference in New Issue
Block a user