khyari hamza
02c75fb4bb
Add structured data builders and Rich renderers for version, info, and diagnostics commands. Support --format rich/plain/json/yaml output parity and --check flag for diagnostics (exits non-zero on errors). Includes Behave scenarios, Robot smoke tests, ASV benchmarks, and updates existing tests to match the new Rich panel output format.
348 lines
12 KiB
Python
348 lines
12 KiB
Python
"""Step definitions for CLI coverage tests."""
|
|
|
|
import subprocess
|
|
import sys
|
|
from contextlib import redirect_stdout
|
|
from io import StringIO
|
|
from unittest.mock import patch
|
|
|
|
from behave import then, when
|
|
from click.exceptions import Exit as ClickExit
|
|
from typer import Exit
|
|
|
|
from cleveragents.cli.main import (
|
|
_print_basic_help,
|
|
app,
|
|
build,
|
|
convert_exit_code,
|
|
ensure_cli_commands_registered,
|
|
main,
|
|
tell,
|
|
)
|
|
|
|
|
|
@when('I run shell command "{command}"')
|
|
def step_run_shell_command(context, command):
|
|
"""Run a CLI command using subprocess shell."""
|
|
try:
|
|
# Check if we have a test directory to run from
|
|
cwd = context.test_dir if hasattr(context, "test_dir") else None
|
|
|
|
# Replace 'cleveragents' with the current Python interpreter running the module
|
|
# This ensures we use the same Python (from nox venv) that has all dependencies
|
|
if command.startswith("cleveragents"):
|
|
# Handle both "cleveragents --arg" and "cleveragents command --arg"
|
|
# Use cleveragents.cli.main since cleveragents.cli is a package without __main__.py
|
|
python_cmd = f'"{sys.executable}" -m cleveragents.cli.main'
|
|
if command == "cleveragents":
|
|
command = python_cmd
|
|
else:
|
|
command = python_cmd + command[len("cleveragents") :]
|
|
|
|
# Use shell=True to handle quoted arguments correctly
|
|
result = subprocess.run(
|
|
command, capture_output=True, text=True, timeout=120, cwd=cwd, shell=True
|
|
)
|
|
context.exit_code = result.returncode
|
|
context.output = result.stdout + result.stderr
|
|
|
|
# Check if command failed due to missing AI provider
|
|
# If so, skip the scenario as this is expected in test environments
|
|
if context.exit_code != 0:
|
|
output = context.output.lower()
|
|
if "no ai provider configured" in output or (
|
|
"ai provider" in output and "not" in output
|
|
):
|
|
context.scenario.skip(
|
|
"Skipping test - no AI provider configured (expected in test env)"
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
context.exit_code = -1
|
|
context.output = "Command timed out"
|
|
except Exception as e:
|
|
context.exit_code = -1
|
|
context.output = str(e)
|
|
|
|
|
|
@then("the shell command should succeed")
|
|
def step_shell_command_succeeds(context):
|
|
"""Check that shell command succeeded."""
|
|
assert context.exit_code == 0, f"Expected exit code 0, got {context.exit_code}"
|
|
|
|
|
|
@then("the command should fail with exit code {code:d}")
|
|
def step_command_fails_with_code(context, code):
|
|
"""Check that command failed with specific code."""
|
|
assert context.exit_code == code, (
|
|
f"Expected exit code {code}, got {context.exit_code}"
|
|
)
|
|
|
|
|
|
@then('the shell output should contain "{text}"')
|
|
def step_shell_output_contains(context, text):
|
|
"""Check that shell output contains text."""
|
|
assert text in context.output, f"'{text}' not found in output: {context.output}"
|
|
|
|
|
|
@when("I call the main function directly with no args")
|
|
def step_call_main_no_args(context):
|
|
"""Call main function with no arguments."""
|
|
with patch("sys.argv", ["cleveragents"]):
|
|
context.exit_code = main()
|
|
|
|
|
|
@when('I call the main function directly with "{args}"')
|
|
def step_call_main_with_args(context, args):
|
|
"""Call main function with specific arguments."""
|
|
argv = ["cleveragents", *args.split()]
|
|
with patch("sys.argv", argv):
|
|
context.exit_code = main()
|
|
|
|
|
|
@then("it should return {code:d}")
|
|
def step_check_return_code(context, code):
|
|
"""Check the return code."""
|
|
assert context.exit_code == code, f"Expected {code}, got {context.exit_code}"
|
|
|
|
|
|
@when("I run a command that gets interrupted")
|
|
def step_run_interrupted_command(context):
|
|
"""Simulate a command that gets interrupted."""
|
|
with patch("cleveragents.cli.main.app") as mock_app:
|
|
mock_app.side_effect = KeyboardInterrupt()
|
|
context.exit_code = main(["info"])
|
|
|
|
|
|
@then("it should handle the KeyboardInterrupt gracefully")
|
|
def step_check_keyboard_interrupt_handled(context):
|
|
"""Check that KeyboardInterrupt was handled."""
|
|
assert context.exit_code == 130, (
|
|
f"Expected 130 for KeyboardInterrupt, got {context.exit_code}"
|
|
)
|
|
|
|
|
|
@when("I run a command that calls sys.exit(1)")
|
|
def step_run_sys_exit_command(context):
|
|
"""Simulate a command that calls sys.exit."""
|
|
with patch("cleveragents.cli.main.app") as mock_app:
|
|
mock_app.side_effect = SystemExit(1)
|
|
context.exit_code = main(["info"])
|
|
|
|
|
|
@then("it should propagate the exit code correctly")
|
|
def step_check_exit_code_propagated(context):
|
|
"""Check that exit code was propagated."""
|
|
assert context.exit_code == 1, f"Expected 1, got {context.exit_code}"
|
|
|
|
|
|
# Additional steps to increase coverage
|
|
|
|
|
|
@when("I test convert_exit_code with various inputs")
|
|
def step_test_convert_exit_code(context):
|
|
"""Test the convert_exit_code function."""
|
|
context.results = []
|
|
# Test with int
|
|
context.results.append(convert_exit_code(0))
|
|
context.results.append(convert_exit_code(1))
|
|
context.results.append(convert_exit_code(-1))
|
|
# Test with None
|
|
context.results.append(convert_exit_code(None))
|
|
# Test with Exit exception
|
|
context.results.append(convert_exit_code(Exit(2)))
|
|
# Test with ClickExit exception
|
|
context.results.append(convert_exit_code(ClickExit(3)))
|
|
# Test with other object
|
|
context.results.append(convert_exit_code("string"))
|
|
|
|
|
|
@then("convert_exit_code should handle all types correctly")
|
|
def step_check_convert_exit_code(context):
|
|
"""Check convert_exit_code results."""
|
|
expected = [0, 1, -1, 0, 2, 3, 1] # string returns 1, not 0
|
|
assert context.results == expected, f"Expected {expected}, got {context.results}"
|
|
|
|
|
|
@when("I test the CLI app commands")
|
|
def step_test_cli_commands(context):
|
|
"""Test CLI app commands directly."""
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
runner = CliRunner()
|
|
|
|
context.results = []
|
|
# Test version
|
|
result = runner.invoke(app, ["--version"])
|
|
context.results.append((result.exit_code, "1.0.0" in result.stdout))
|
|
|
|
# Test help
|
|
result = runner.invoke(app, ["--help"])
|
|
context.results.append((result.exit_code, "Usage" in result.stdout))
|
|
|
|
# Test info
|
|
result = runner.invoke(app, ["info"])
|
|
context.results.append((result.exit_code, "Environment" in result.stdout))
|
|
|
|
# Test diagnostics
|
|
result = runner.invoke(app, ["diagnostics"])
|
|
context.results.append((result.exit_code, "Checks" in result.stdout))
|
|
|
|
# Test init - use temp directory to avoid conflicts
|
|
import os
|
|
|
|
from cleveragents.application.container import reset_container
|
|
from cleveragents.config.settings import Settings
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
orig_cwd = Path.cwd()
|
|
try:
|
|
# Reset container and settings so init uses the temp directory's
|
|
# database path instead of any previously-cached state
|
|
reset_container()
|
|
Settings._instance = None
|
|
|
|
os.chdir(tmpdir)
|
|
result = runner.invoke(app, ["init", "testproj"])
|
|
context.results.append(
|
|
(result.exit_code, "initialized successfully" in result.stdout)
|
|
)
|
|
finally:
|
|
os.chdir(orig_cwd)
|
|
# Reset again so subsequent tests don't use temp directory state
|
|
reset_container()
|
|
Settings._instance = None
|
|
|
|
|
|
@then("all CLI commands should work correctly")
|
|
def step_check_cli_commands(context):
|
|
"""Check CLI command results."""
|
|
for idx, (exit_code, output_check) in enumerate(context.results):
|
|
# All commands should succeed now
|
|
assert exit_code == 0, f"Command {idx} failed with exit code {exit_code}"
|
|
assert output_check, f"Command {idx} output check failed"
|
|
|
|
|
|
@when("I test exception handling in main")
|
|
def step_test_exception_handling(context):
|
|
"""Test exception handling in main function."""
|
|
context.results = []
|
|
|
|
# Test CleverAgentsError
|
|
from cleveragents.core.exceptions import CleverAgentsError
|
|
|
|
with patch("cleveragents.cli.main.app") as mock_app:
|
|
mock_app.side_effect = CleverAgentsError("Test error")
|
|
result = main(["info"])
|
|
context.results.append(result)
|
|
|
|
# Test generic Exception
|
|
with patch("cleveragents.cli.main.app") as mock_app:
|
|
mock_app.side_effect = Exception("Generic error")
|
|
result = main(["info"])
|
|
context.results.append(result)
|
|
|
|
# Test Exit with code
|
|
with patch("cleveragents.cli.main.app") as mock_app:
|
|
mock_app.side_effect = Exit(5)
|
|
result = main(["info"])
|
|
context.results.append(result)
|
|
|
|
|
|
@then("exception handling should return correct codes")
|
|
def step_check_exception_handling(context):
|
|
"""Check exception handling results."""
|
|
expected = [1, 1, 5]
|
|
assert context.results == expected, f"Expected {expected}, got {context.results}"
|
|
|
|
|
|
@when("I trigger the basic CLI help output")
|
|
def step_trigger_basic_help_output(context):
|
|
"""Capture the lightweight CLI help output."""
|
|
buffer = StringIO()
|
|
with redirect_stdout(buffer):
|
|
_print_basic_help()
|
|
context.basic_help_output = buffer.getvalue()
|
|
|
|
|
|
@then("the basic help output should list common commands")
|
|
def step_check_basic_help_output(context):
|
|
"""Verify the lightweight help includes key commands."""
|
|
output = context.basic_help_output
|
|
assert "CleverAgents - AI-powered development assistant" in output
|
|
for keyword in ["project", "context", "plan", "auto-debug", "version"]:
|
|
assert keyword in output, f"{keyword} missing from help output"
|
|
|
|
|
|
@when("I ensure CLI subcommands are registered lazily")
|
|
def step_ensure_cli_commands_registered(context):
|
|
"""Ensure subcommand registration sets the flag once."""
|
|
import importlib
|
|
|
|
cli_main = importlib.import_module("cleveragents.cli.main")
|
|
original_state = cli_main._subcommands_registered
|
|
cli_main._subcommands_registered = False
|
|
try:
|
|
ensure_cli_commands_registered()
|
|
first_state = cli_main._subcommands_registered
|
|
ensure_cli_commands_registered()
|
|
second_state = cli_main._subcommands_registered
|
|
finally:
|
|
cli_main._subcommands_registered = original_state
|
|
|
|
context.registration_states = (first_state, second_state)
|
|
|
|
|
|
@then("the registration flag should stay enabled")
|
|
def step_check_registration_flag(context):
|
|
"""Check registration state is preserved."""
|
|
first_state, second_state = context.registration_states
|
|
assert first_state is True
|
|
assert second_state is True
|
|
|
|
|
|
@when("I invoke the tell shortcut with an actor override")
|
|
def step_call_tell_with_overrides(context):
|
|
"""Invoke tell shortcut ensuring actor override is forwarded."""
|
|
with patch("cleveragents.cli.commands.plan.tell") as mock_tell:
|
|
tell(
|
|
prompt="Explain coverage improvements",
|
|
name="coverage-plan",
|
|
actor="coverage-actor",
|
|
stream=True,
|
|
)
|
|
context.tell_kwargs = mock_tell.call_args.kwargs
|
|
|
|
|
|
@then("the tell shortcut should forward the actor override")
|
|
def step_check_tell_overrides(context):
|
|
"""Validate tell forwards actor override."""
|
|
expected = {
|
|
"prompt": "Explain coverage improvements",
|
|
"stream": True,
|
|
"name": "coverage-plan",
|
|
"actor": "coverage-actor",
|
|
}
|
|
assert context.tell_kwargs == expected, (
|
|
f"Expected {expected}, got {context.tell_kwargs}"
|
|
)
|
|
|
|
|
|
@when("I invoke the build shortcut with an actor override")
|
|
def step_call_build_with_overrides(context):
|
|
"""Invoke build shortcut ensuring actor override is forwarded."""
|
|
with patch("cleveragents.cli.commands.plan.build") as mock_build:
|
|
build(verbose=True, actor="build-actor")
|
|
context.build_kwargs = mock_build.call_args.kwargs
|
|
|
|
|
|
@then("the build shortcut should forward the actor override")
|
|
def step_check_build_overrides(context):
|
|
"""Validate build forwards actor override."""
|
|
expected = {"verbose": True, "actor": "build-actor"}
|
|
assert context.build_kwargs == expected, (
|
|
f"Expected {expected}, got {context.build_kwargs}"
|
|
)
|