cleveragents-core/robot/helper_tdd_validation_required_flag.py

"""Helper script for tdd_validation_required_flag.robot smoke tests.

Each subcommand exercises the ``agents validation add`` CLI path via
``typer.testing.CliRunner`` to reproduce bug #1038.  The specification
(``docs/specification.md`` line 22339) states that the validation mode can
be set "via ``--required``/``--informational`` on ``agents validation add``",
and numerous workflow examples in the spec use the ``--required`` flag.

Bug #1038 reported that the ``add`` command did not accept these flags,
causing a ``NoSuchOption`` error.  The fix adds ``--required`` and
``--informational`` as mutually exclusive boolean options that override the
``mode`` field in the YAML config when specified.

The helper reports the **real** outcome: it exits 0 and prints the sentinel
when the expected behaviour is observed (bug fixed), and exits 1 when the
bug is still present.

This test was written to capture bug #1038 per ticket #1102.
"""

from __future__ import annotations

import contextlib
import os
import sys
import tempfile
from collections.abc import Callable
from pathlib import Path
from typing import NoReturn
from unittest.mock import MagicMock, patch

# Ensure local source tree is importable.
_ROOT: Path = Path(__file__).resolve().parents[1]
_SRC: str = str(_ROOT / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

from typer.testing import CliRunner  # noqa: E402

from cleveragents.cli.commands.validation import app as validation_app  # noqa: E402

runner: CliRunner = CliRunner()

_PATCH_SVC: str = "cleveragents.cli.commands.validation._get_tool_registry_service"


def _fail(message: str) -> NoReturn:
    """Print an error message to stderr and exit with code 1."""
    print(message, file=sys.stderr)
    sys.exit(1)


def _safe_unlink(path: str) -> None:
    """Remove a file if it exists, silently ignoring missing files."""
    with contextlib.suppress(FileNotFoundError):
        os.unlink(path)


def _create_yaml_config(mode: str = "required") -> str:
    """Create a temporary YAML config file with the given mode.

    Uses a flat ``mode`` key to match what ``Validation.from_config()``
    actually reads (``config.get("mode", "required")``).
    """
    config_content: str = (
        "name: local/unit-tests\n"
        "description: Unit tests\n"
        "source: custom\n"
        f"mode: {mode}\n"
        "code: |\n"
        "  def run(inputs):\n"
        "      return {'passed': True}\n"
    )
    with tempfile.NamedTemporaryFile(
        mode="w",
        suffix=".yaml",
        delete=False,
    ) as tmp:
        tmp.write(config_content)
        return tmp.name


# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------


def _check_required() -> None:
    """Invoke ``validation add --config <file> --required`` and verify accepted.

    Exits 0 with sentinel when ``--required`` is recognised (bug fixed).
    Exits 1 when Typer rejects the flag (bug still present).
    """
    config_path: str = _create_yaml_config()
    try:
        mock_service: MagicMock = MagicMock()
        mock_service.register_tool.side_effect = lambda v: v

        with patch(_PATCH_SVC, return_value=mock_service):
            result = runner.invoke(
                validation_app,
                ["add", "--config", config_path, "--required", "--format", "plain"],
            )

        if "No such option" in result.output or result.exit_code == 2:
            _fail(
                f"validation add rejected --required flag.\n"
                f"Exit code: {result.exit_code}\n"
                f"Output: {result.output}"
            )
        if result.exit_code != 0:
            _fail(
                f"validation add --required exited with code {result.exit_code}.\n"
                f"Output: {result.output}"
            )
        print("tdd-validation-required-flag-ok")
    finally:
        _safe_unlink(config_path)


def _check_informational() -> None:
    """Invoke ``validation add --config <file> --informational`` and verify accepted.

    Exits 0 with sentinel when ``--informational`` is recognised (bug fixed).
    Exits 1 when Typer rejects the flag (bug still present).
    """
    config_path: str = _create_yaml_config()
    try:
        mock_service: MagicMock = MagicMock()
        mock_service.register_tool.side_effect = lambda v: v

        with patch(_PATCH_SVC, return_value=mock_service):
            result = runner.invoke(
                validation_app,
                [
                    "add",
                    "--config",
                    config_path,
                    "--informational",
                    "--format",
                    "plain",
                ],
            )

        if "No such option" in result.output or result.exit_code == 2:
            _fail(
                f"validation add rejected --informational flag.\n"
                f"Exit code: {result.exit_code}\n"
                f"Output: {result.output}"
            )
        if result.exit_code != 0:
            _fail(
                f"validation add --informational exited with code {result.exit_code}.\n"
                f"Output: {result.output}"
            )
        print("tdd-validation-informational-flag-ok")
    finally:
        _safe_unlink(config_path)


def _check_required_overrides_config() -> None:
    """Invoke with ``--required`` on a YAML that has ``mode: informational``.

    Verifies that the ``--required`` CLI flag overrides the YAML config mode.
    Exits 0 with sentinel when the override works (bug fixed).
    Exits 1 when the flag is not recognised (bug still present).
    """
    config_path: str = _create_yaml_config(mode="informational")
    try:
        mock_service: MagicMock = MagicMock()
        mock_service.register_tool.side_effect = lambda v: v

        with patch(_PATCH_SVC, return_value=mock_service):
            result = runner.invoke(
                validation_app,
                ["add", "--config", config_path, "--required", "--format", "plain"],
            )

        if "No such option" in result.output or result.exit_code == 2:
            _fail(
                f"validation add rejected --required flag.\n"
                f"Exit code: {result.exit_code}\n"
                f"Output: {result.output}"
            )
        if result.exit_code != 0:
            _fail(
                f"validation add --required exited with code {result.exit_code}.\n"
                f"Output: {result.output}"
            )

        # Verify the mode was overridden to "required"
        if "mode: required" not in result.output:
            _fail(
                f"--required flag did not override YAML config mode.\n"
                f"Expected 'mode: required' in output.\n"
                f"Output: {result.output}"
            )

        # Verify the service layer received the correct mode
        if mock_service.register_tool.called:
            call_args = mock_service.register_tool.call_args
            validation_arg = call_args[0][0]
            actual_mode = getattr(validation_arg, "mode", None)
            if actual_mode != "required":
                _fail(
                    f"register_tool received mode '{actual_mode}' instead of "
                    f"'required'. The --required flag did not override the "
                    f"YAML config mode at the service layer."
                )

        print("tdd-validation-required-overrides-config-ok")
    finally:
        _safe_unlink(config_path)


def _check_informational_overrides_config() -> None:
    """Invoke with ``--informational`` on a YAML that has ``mode: required``.

    Verifies that the ``--informational`` CLI flag overrides the YAML config
    mode.  Exits 0 with sentinel when the override works (bug fixed).
    Exits 1 when the flag is not recognised (bug still present).
    """
    config_path: str = _create_yaml_config(mode="required")
    try:
        mock_service: MagicMock = MagicMock()
        mock_service.register_tool.side_effect = lambda v: v

        with patch(_PATCH_SVC, return_value=mock_service):
            result = runner.invoke(
                validation_app,
                [
                    "add",
                    "--config",
                    config_path,
                    "--informational",
                    "--format",
                    "plain",
                ],
            )

        if "No such option" in result.output or result.exit_code == 2:
            _fail(
                f"validation add rejected --informational flag.\n"
                f"Exit code: {result.exit_code}\n"
                f"Output: {result.output}"
            )
        if result.exit_code != 0:
            _fail(
                f"validation add --informational exited with code "
                f"{result.exit_code}.\n"
                f"Output: {result.output}"
            )

        # Verify the mode was overridden to "informational"
        if "mode: informational" not in result.output:
            _fail(
                f"--informational flag did not override YAML config mode.\n"
                f"Expected 'mode: informational' in output.\n"
                f"Output: {result.output}"
            )

        # Verify the service layer received the correct mode
        if mock_service.register_tool.called:
            call_args = mock_service.register_tool.call_args
            validation_arg = call_args[0][0]
            actual_mode = getattr(validation_arg, "mode", None)
            if actual_mode != "informational":
                _fail(
                    f"register_tool received mode '{actual_mode}' instead of "
                    f"'informational'. The --informational flag did not override "
                    f"the YAML config mode at the service layer."
                )

        print("tdd-validation-informational-overrides-config-ok")
    finally:
        _safe_unlink(config_path)


def _check_both_flags_rejected() -> None:
    """Invoke with both ``--required`` and ``--informational`` simultaneously.

    These flags are mutually exclusive; the CLI should reject the invocation
    and abort.  Exits 0 with sentinel when the rejection occurs correctly.
    Exits 1 when the CLI does not reject the combination.
    """
    config_path: str = _create_yaml_config()
    try:
        mock_service: MagicMock = MagicMock()
        mock_service.register_tool.side_effect = lambda v: v

        with patch(_PATCH_SVC, return_value=mock_service):
            result = runner.invoke(
                validation_app,
                [
                    "add",
                    "--config",
                    config_path,
                    "--required",
                    "--informational",
                    "--format",
                    "plain",
                ],
            )

        if result.exit_code == 0:
            _fail(
                f"validation add accepted both --required and --informational "
                f"(should have been rejected).\n"
                f"Output: {result.output}"
            )

        if "mutually exclusive" not in result.output:
            _fail(
                f"Expected 'mutually exclusive' in error output.\n"
                f"Output: {result.output}"
            )

        print("tdd-validation-both-flags-rejected-ok")
    finally:
        _safe_unlink(config_path)


# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------

_COMMANDS: dict[str, Callable[[], None]] = {
    "check-required": _check_required,
    "check-informational": _check_informational,
    "check-required-overrides-config": _check_required_overrides_config,
    "check-informational-overrides-config": _check_informational_overrides_config,
    "check-both-flags-rejected": _check_both_flags_rejected,
}

if __name__ == "__main__":
    if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
        print(
            f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>",
            file=sys.stderr,
        )
        sys.exit(1)
    cmd: Callable[[], None] = _COMMANDS[sys.argv[1]]
    cmd()