forked from HAL9000/cleveragents-core
506 lines
19 KiB
Python
506 lines
19 KiB
Python
"""Step definitions for quality gates checker script coverage tests.
|
|
|
|
These steps test all functions in scripts/check-quality-gates.py by mocking
|
|
subprocess.run to return controlled results, achieving full line and branch
|
|
coverage.
|
|
"""
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
|
|
def _load_quality_gates_module():
|
|
"""Load the check-quality-gates.py script as a module via importlib."""
|
|
spec = importlib.util.spec_from_file_location(
|
|
"check_quality_gates", "scripts/check-quality-gates.py"
|
|
)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
_qg = _load_quality_gates_module()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_coverage: Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("the coverage tool reports a total of {percent} percent")
|
|
def step_coverage_tool_reports_percent(context: Context, percent: str) -> None:
|
|
"""Mock subprocess.run so coverage report outputs a numeric total."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = f"{percent}\n"
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given('the coverage tool returns a non-zero exit code with stderr "{stderr_msg}"')
|
|
def step_coverage_tool_nonzero_exit(context: Context, stderr_msg: str) -> None:
|
|
"""Mock subprocess.run so coverage report fails with non-zero exit."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 1
|
|
mock_result.stdout = ""
|
|
mock_result.stderr = stderr_msg
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given('the coverage tool returns unparseable output "{output}"')
|
|
def step_coverage_tool_unparseable(context: Context, output: str) -> None:
|
|
"""Mock subprocess.run so coverage report returns unparseable output."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = output
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_coverage: When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the quality gates check_coverage function is called with minimum {min_cov:d}")
|
|
def step_call_check_coverage(context: Context, min_cov: int) -> None:
|
|
"""Call check_coverage with the prepared mock."""
|
|
with patch("subprocess.run", return_value=context.qg_mock_result):
|
|
passed, msg = _qg.check_coverage(min_cov)
|
|
context.qg_passed = passed
|
|
context.qg_message = msg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_coverage: Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates coverage result should be passed")
|
|
def step_coverage_result_passed(context: Context) -> None:
|
|
assert context.qg_passed is True, (
|
|
f"Expected coverage to pass, but it failed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then("the quality gates coverage result should be failed")
|
|
def step_coverage_result_failed(context: Context) -> None:
|
|
assert context.qg_passed is False, (
|
|
f"Expected coverage to fail, but it passed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then('the quality gates coverage message should contain "{expected}"')
|
|
def step_coverage_message_contains(context: Context, expected: str) -> None:
|
|
assert expected in context.qg_message, (
|
|
f"Expected '{expected}' in message, got: {context.qg_message}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_typecheck: Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("pyright reports JSON output with zero errors")
|
|
def step_pyright_zero_errors(context: Context) -> None:
|
|
"""Mock pyright returning JSON with zero errors."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = json.dumps({"summary": {"errorCount": 0}})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("pyright reports JSON output with {count:d} errors")
|
|
def step_pyright_with_errors(context: Context, count: int) -> None:
|
|
"""Mock pyright returning JSON with some errors."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 1
|
|
mock_result.stdout = json.dumps({"summary": {"errorCount": count}})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("pyright returns invalid JSON output with exit code {code:d}")
|
|
def step_pyright_invalid_json(context: Context, code: int) -> None:
|
|
"""Mock pyright returning non-JSON output."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = code
|
|
mock_result.stdout = "not valid json at all"
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_typecheck: When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the quality gates check_typecheck function is called")
|
|
def step_call_check_typecheck(context: Context) -> None:
|
|
"""Call check_typecheck with the prepared mock."""
|
|
with patch("subprocess.run", return_value=context.qg_mock_result):
|
|
passed, msg = _qg.check_typecheck()
|
|
context.qg_passed = passed
|
|
context.qg_message = msg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_typecheck: Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates typecheck result should be passed")
|
|
def step_typecheck_result_passed(context: Context) -> None:
|
|
assert context.qg_passed is True, (
|
|
f"Expected typecheck to pass, but it failed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then("the quality gates typecheck result should be failed")
|
|
def step_typecheck_result_failed(context: Context) -> None:
|
|
assert context.qg_passed is False, (
|
|
f"Expected typecheck to fail, but it passed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then('the quality gates typecheck message should contain "{expected}"')
|
|
def step_typecheck_message_contains(context: Context, expected: str) -> None:
|
|
assert expected in context.qg_message, (
|
|
f"Expected '{expected}' in message, got: {context.qg_message}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_security: Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("bandit reports JSON output with zero results")
|
|
def step_bandit_zero_results(context: Context) -> None:
|
|
"""Mock bandit returning JSON with no results."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = json.dumps({"results": []})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("bandit reports JSON output with {count:d} high-severity results")
|
|
def step_bandit_with_results(context: Context, count: int) -> None:
|
|
"""Mock bandit returning JSON with high-severity results."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 1
|
|
results = [{"issue_text": f"Issue {i}"} for i in range(count)]
|
|
mock_result.stdout = json.dumps({"results": results})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("bandit returns invalid JSON output with exit code {code:d}")
|
|
def step_bandit_invalid_json(context: Context, code: int) -> None:
|
|
"""Mock bandit returning non-JSON output."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = code
|
|
mock_result.stdout = "bandit crashed or produced no json"
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_security: When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the quality gates check_security function is called")
|
|
def step_call_check_security(context: Context) -> None:
|
|
"""Call check_security with the prepared mock."""
|
|
with patch("subprocess.run", return_value=context.qg_mock_result):
|
|
passed, msg = _qg.check_security()
|
|
context.qg_passed = passed
|
|
context.qg_message = msg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_security: Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates security result should be passed")
|
|
def step_security_result_passed(context: Context) -> None:
|
|
assert context.qg_passed is True, (
|
|
f"Expected security to pass, but it failed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then("the quality gates security result should be failed")
|
|
def step_security_result_failed(context: Context) -> None:
|
|
assert context.qg_passed is False, (
|
|
f"Expected security to fail, but it passed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then('the quality gates security message should contain "{expected}"')
|
|
def step_security_message_contains(context: Context, expected: str) -> None:
|
|
assert expected in context.qg_message, (
|
|
f"Expected '{expected}' in message, got: {context.qg_message}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_dead_code: Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("vulture returns exit code 0 with no output")
|
|
def step_vulture_clean(context: Context) -> None:
|
|
"""Mock vulture returning success with no dead code."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = ""
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("vulture returns exit code 1 with {count:d} lines of dead code output")
|
|
def step_vulture_dead_code(context: Context, count: int) -> None:
|
|
"""Mock vulture returning failure with dead code lines."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 1
|
|
lines = [
|
|
f"src/cleveragents/foo.py:{i}: unused function 'bar{i}'"
|
|
for i in range(1, count + 1)
|
|
]
|
|
mock_result.stdout = "\n".join(lines)
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_dead_code: When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the quality gates check_dead_code function is called")
|
|
def step_call_check_dead_code(context: Context) -> None:
|
|
"""Call check_dead_code with the prepared mock."""
|
|
with patch("subprocess.run", return_value=context.qg_mock_result):
|
|
passed, msg = _qg.check_dead_code()
|
|
context.qg_passed = passed
|
|
context.qg_message = msg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_dead_code: Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates dead code result should be passed")
|
|
def step_dead_code_result_passed(context: Context) -> None:
|
|
assert context.qg_passed is True, (
|
|
f"Expected dead code check to pass, but it failed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then("the quality gates dead code result should be failed")
|
|
def step_dead_code_result_failed(context: Context) -> None:
|
|
assert context.qg_passed is False, (
|
|
f"Expected dead code check to fail, but it passed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then('the quality gates dead code message should contain "{expected}"')
|
|
def step_dead_code_message_contains(context: Context, expected: str) -> None:
|
|
assert expected in context.qg_message, (
|
|
f"Expected '{expected}' in message, got: {context.qg_message}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_complexity: Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("radon reports JSON output with zero complex blocks")
|
|
def step_radon_zero_blocks(context: Context) -> None:
|
|
"""Mock radon returning JSON with no complex blocks."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = json.dumps({})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("radon reports JSON output with {count:d} complex blocks")
|
|
def step_radon_with_blocks(context: Context, count: int) -> None:
|
|
"""Mock radon returning JSON with complex blocks."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
blocks = [{"name": f"func_{i}", "complexity": 50} for i in range(count)]
|
|
mock_result.stdout = json.dumps({"src/cleveragents/foo.py": blocks})
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
@given("radon returns invalid JSON output")
|
|
def step_radon_invalid_json(context: Context) -> None:
|
|
"""Mock radon returning non-JSON output."""
|
|
mock_result = MagicMock()
|
|
mock_result.returncode = 0
|
|
mock_result.stdout = "not json"
|
|
mock_result.stderr = ""
|
|
context.qg_mock_result = mock_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_complexity: When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('the quality gates check_complexity function is called with max grade "{grade}"')
|
|
def step_call_check_complexity(context: Context, grade: str) -> None:
|
|
"""Call check_complexity with the prepared mock."""
|
|
with patch("subprocess.run", return_value=context.qg_mock_result):
|
|
passed, msg = _qg.check_complexity(grade)
|
|
context.qg_passed = passed
|
|
context.qg_message = msg
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# check_complexity: Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates complexity result should be passed")
|
|
def step_complexity_result_passed(context: Context) -> None:
|
|
assert context.qg_passed is True, (
|
|
f"Expected complexity check to pass, but it failed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then("the quality gates complexity result should be failed")
|
|
def step_complexity_result_failed(context: Context) -> None:
|
|
assert context.qg_passed is False, (
|
|
f"Expected complexity check to fail, but it passed: {context.qg_message}"
|
|
)
|
|
|
|
|
|
@then('the quality gates complexity message should contain "{expected}"')
|
|
def step_complexity_message_contains(context: Context, expected: str) -> None:
|
|
assert expected in context.qg_message, (
|
|
f"Expected '{expected}' in message, got: {context.qg_message}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# main(): Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("all quality gate functions are mocked to return success")
|
|
def step_all_gates_success(context: Context) -> None:
|
|
"""Prepare mocks so all gate functions return (True, ...)."""
|
|
context.qg_gate_mocks = {
|
|
"check_coverage": (True, "Coverage: 90.0% (minimum: 85%)"),
|
|
"check_typecheck": (True, "Type checking: 0 errors"),
|
|
"check_security": (True, "Security: 0 high-severity issues"),
|
|
"check_dead_code": (True, "Dead code: none detected"),
|
|
"check_complexity": (True, "Complexity: no blocks with grade >= F"),
|
|
}
|
|
|
|
|
|
@given("the quality gate coverage function is mocked to return failure")
|
|
def step_coverage_gate_fails(context: Context) -> None:
|
|
"""Prepare mocks so coverage fails but others pass."""
|
|
context.qg_gate_mocks = {
|
|
"check_coverage": (False, "Coverage: 50.0% (minimum: 85%)"),
|
|
"check_typecheck": (True, "Type checking: 0 errors"),
|
|
"check_security": (True, "Security: 0 high-severity issues"),
|
|
"check_dead_code": (True, "Dead code: none detected"),
|
|
"check_complexity": (True, "Complexity: no blocks with grade >= F"),
|
|
}
|
|
|
|
|
|
@given("a temporary working directory with no build subdirectory")
|
|
def step_temp_dir_no_build(context: Context) -> None:
|
|
"""Create a temp directory and store it on context."""
|
|
context.qg_tmpdir = tempfile.mkdtemp()
|
|
context.qg_gate_mocks = {
|
|
"check_coverage": (True, "Coverage: 90.0% (minimum: 85%)"),
|
|
"check_typecheck": (True, "Type checking: 0 errors"),
|
|
"check_security": (True, "Security: 0 high-severity issues"),
|
|
"check_dead_code": (True, "Dead code: none detected"),
|
|
"check_complexity": (True, "Complexity: no blocks with grade >= F"),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# main(): When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the quality gates main function is invoked with default arguments")
|
|
def step_call_main_defaults(context: Context) -> None:
|
|
"""Call main() with all gate functions mocked."""
|
|
mocks = context.qg_gate_mocks
|
|
with (
|
|
patch.object(_qg, "check_coverage", return_value=mocks["check_coverage"]),
|
|
patch.object(_qg, "check_typecheck", return_value=mocks["check_typecheck"]),
|
|
patch.object(_qg, "check_security", return_value=mocks["check_security"]),
|
|
patch.object(_qg, "check_dead_code", return_value=mocks["check_dead_code"]),
|
|
patch.object(_qg, "check_complexity", return_value=mocks["check_complexity"]),
|
|
patch("sys.argv", ["check-quality-gates.py"]),
|
|
):
|
|
context.qg_main_exit_code = _qg.main()
|
|
|
|
|
|
@when("the quality gates main function is invoked from that temporary directory")
|
|
def step_call_main_from_tmpdir(context: Context) -> None:
|
|
"""Call main() from a temp directory to verify build dir creation."""
|
|
mocks = context.qg_gate_mocks
|
|
old_cwd = os.getcwd()
|
|
os.chdir(context.qg_tmpdir)
|
|
try:
|
|
with (
|
|
patch.object(_qg, "check_coverage", return_value=mocks["check_coverage"]),
|
|
patch.object(_qg, "check_typecheck", return_value=mocks["check_typecheck"]),
|
|
patch.object(_qg, "check_security", return_value=mocks["check_security"]),
|
|
patch.object(_qg, "check_dead_code", return_value=mocks["check_dead_code"]),
|
|
patch.object(
|
|
_qg, "check_complexity", return_value=mocks["check_complexity"]
|
|
),
|
|
patch("sys.argv", ["check-quality-gates.py"]),
|
|
):
|
|
context.qg_main_exit_code = _qg.main()
|
|
finally:
|
|
os.chdir(old_cwd)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# main(): Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the quality gates main should return exit code {code:d}")
|
|
def step_main_exit_code(context: Context, code: int) -> None:
|
|
assert context.qg_main_exit_code == code, (
|
|
f"Expected exit code {code}, got {context.qg_main_exit_code}"
|
|
)
|
|
|
|
|
|
@then("a build directory should exist in the temporary working directory")
|
|
def step_build_dir_exists(context: Context) -> None:
|
|
from pathlib import Path
|
|
|
|
build_path = Path(context.qg_tmpdir) / "build"
|
|
assert build_path.is_dir(), (
|
|
f"Expected build directory at {build_path}, but it does not exist"
|
|
)
|