forked from HAL9000/cleveragents-core
c8cd7eab82
Implemented the three-tag TDD bug-capture system in Behave environment hooks: - Added tag validation in before_scenario: @tdd_bug_<N> requires @tdd_bug, @tdd_expected_fail requires both @tdd_bug and @tdd_bug_<N> - Added result inversion via Scenario.run() wrapper installed in before_all: scenarios tagged @tdd_expected_fail that fail are reported as passed (expected failure), and scenarios that unexpectedly pass are reported as failed with guidance to remove the tag - Added helper functions validate_tdd_tags() and should_invert_result() - Added inline documentation referencing CONTRIBUTING.md > TDD Bug Test Tags - Added Behave test scenarios for tag validation and inversion behavior - Extract apply_tdd_inversion() as a public, testable function that encapsulates all inversion logic with proper guards - Refactored handle_tdd_expected_fail() to delegate to apply_tdd_inversion() after tag validation, eliminating ~55 lines of duplicated inversion logic - Tag validation errors in handle_tdd_expected_fail() are now logged at WARNING level instead of being silently swallowed - Add hook_failed guard: never invert infrastructure/hook errors - Add was_dry_run guard: skip inversion when no test actually ran - Add non-AssertionError guard: warn and skip inversion for exceptions that likely indicate infrastructure problems, not the captured bug - Log exception details at DEBUG level before clearing (previously discarded silently) - Attach synthetic AssertionError to last step on unexpected pass so the failure reason appears in standard Behave formatter output ISSUES CLOSED: #627
288 lines
11 KiB
Python
288 lines
11 KiB
Python
"""Step definitions for the TDD expected-fail handler infrastructure tests.
|
|
|
|
These scenarios exercise both ``apply_tdd_inversion`` (the production code
|
|
path called by the ``Scenario.run()`` monkey-patch) and
|
|
``handle_tdd_expected_fail`` (the standalone entry point) using real Behave
|
|
``Scenario`` and ``Step`` objects to verify scenario-level and **step-level**
|
|
status inversion, including guard paths (hook errors, dry-run, non-assertion
|
|
exceptions).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from behave import given, then, when
|
|
from behave.model import Scenario, Status, Step
|
|
from behave.runner import Context
|
|
|
|
from features.environment import (
|
|
_install_tdd_expected_fail_patch,
|
|
apply_tdd_inversion,
|
|
before_scenario,
|
|
handle_tdd_expected_fail,
|
|
)
|
|
|
|
_STATUS_MAP: dict[str, Status] = {
|
|
"failed": Status.failed,
|
|
"passed": Status.passed,
|
|
"skipped": Status.skipped,
|
|
"untested": Status.untested,
|
|
}
|
|
|
|
|
|
@given('a mock scenario tagged "{tags}" with status "{status}"')
|
|
def step_mock_scenario(context: Context, tags: str, status: str) -> None:
|
|
"""Build a real ``Scenario`` object with the requested tags and status."""
|
|
tag_list = [t.lstrip("@") for t in tags.split()]
|
|
scenario = Scenario(
|
|
filename="<infrastructure-test>",
|
|
line=1,
|
|
keyword="Scenario",
|
|
name="mock-tdd-scenario",
|
|
tags=tag_list,
|
|
)
|
|
# Force the desired starting status.
|
|
scenario.clear_status()
|
|
scenario.set_status(_STATUS_MAP[status])
|
|
# Initialise guard attributes (Behave sets these during Scenario.run()).
|
|
scenario.hook_failed = False
|
|
scenario.was_dry_run = False
|
|
context.mock_scenario = scenario
|
|
context.mock_steps = {}
|
|
|
|
|
|
@given('a mock scenario tagged "{tags}" with status "{status}" and hook_failed')
|
|
def step_mock_scenario_hook_failed(context: Context, tags: str, status: str) -> None:
|
|
"""Build a real ``Scenario`` with ``hook_failed = True``."""
|
|
step_mock_scenario(context, tags, status)
|
|
context.mock_scenario.hook_failed = True
|
|
|
|
|
|
@given('a mock scenario tagged "{tags}" with status "{status}" and was_dry_run')
|
|
def step_mock_scenario_dry_run(context: Context, tags: str, status: str) -> None:
|
|
"""Build a real ``Scenario`` with ``was_dry_run = True``."""
|
|
step_mock_scenario(context, tags, status)
|
|
context.mock_scenario.was_dry_run = True
|
|
|
|
|
|
@given('the mock scenario has a step "{step_name}" with status "{status}"')
|
|
def step_add_mock_step(context: Context, step_name: str, status: str) -> None:
|
|
"""Add a real ``Step`` to the mock scenario with the requested status."""
|
|
step = Step(
|
|
filename="<infrastructure-test>",
|
|
line=1,
|
|
keyword="Given",
|
|
step_type="given",
|
|
name=step_name,
|
|
)
|
|
step.status = _STATUS_MAP[status]
|
|
# Provide an error_message for failed steps so logging can use it.
|
|
if status == "failed":
|
|
step.error_message = f"simulated failure in {step_name!r}"
|
|
step.exception = AssertionError(f"simulated failure in {step_name!r}")
|
|
context.mock_scenario.steps.append(step)
|
|
context.mock_steps[step_name] = step
|
|
|
|
|
|
@given(
|
|
'the mock scenario has an infrastructure-error step "{step_name}"'
|
|
' with exception "{exc_type}"'
|
|
)
|
|
def step_add_mock_step_with_exception(
|
|
context: Context, step_name: str, exc_type: str
|
|
) -> None:
|
|
"""Add a real ``Step`` with a specific non-assertion exception type."""
|
|
step = Step(
|
|
filename="<infrastructure-test>",
|
|
line=1,
|
|
keyword="Given",
|
|
step_type="given",
|
|
name=step_name,
|
|
)
|
|
step.status = Status.failed
|
|
exc_classes: dict[str, type[BaseException]] = {
|
|
"RuntimeError": RuntimeError,
|
|
"TypeError": TypeError,
|
|
"ConnectionError": ConnectionError,
|
|
"AssertionError": AssertionError,
|
|
}
|
|
exc_cls = exc_classes.get(exc_type, RuntimeError)
|
|
step.exception = exc_cls(f"simulated {exc_type} in {step_name!r}")
|
|
step.error_message = f"simulated {exc_type} in {step_name!r}"
|
|
context.mock_scenario.steps.append(step)
|
|
context.mock_steps[step_name] = step
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# apply_tdd_inversion steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("apply_tdd_inversion processes the scenario with failed True")
|
|
def step_run_apply_inversion_failed(context: Context) -> None:
|
|
"""Invoke ``apply_tdd_inversion`` with ``failed=True``."""
|
|
context.apply_inversion_result = apply_tdd_inversion(
|
|
context.mock_scenario, failed=True
|
|
)
|
|
|
|
|
|
@when("apply_tdd_inversion processes the scenario with failed False")
|
|
def step_run_apply_inversion_not_failed(context: Context) -> None:
|
|
"""Invoke ``apply_tdd_inversion`` with ``failed=False``."""
|
|
context.apply_inversion_result = apply_tdd_inversion(
|
|
context.mock_scenario, failed=False
|
|
)
|
|
|
|
|
|
@then("the apply_tdd_inversion result should be False")
|
|
def step_check_inversion_result_false(context: Context) -> None:
|
|
"""Assert ``apply_tdd_inversion`` returned ``False``."""
|
|
assert context.apply_inversion_result is False, (
|
|
f"Expected apply_tdd_inversion to return False, "
|
|
f"got {context.apply_inversion_result!r}"
|
|
)
|
|
|
|
|
|
@then("the apply_tdd_inversion result should be True")
|
|
def step_check_inversion_result_true(context: Context) -> None:
|
|
"""Assert ``apply_tdd_inversion`` returned ``True``."""
|
|
assert context.apply_inversion_result is True, (
|
|
f"Expected apply_tdd_inversion to return True, "
|
|
f"got {context.apply_inversion_result!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# handle_tdd_expected_fail steps (standalone entry point)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the TDD expected-fail handler processes the scenario")
|
|
def step_run_handler(context: Context) -> None:
|
|
"""Invoke ``handle_tdd_expected_fail`` on the mock scenario."""
|
|
handle_tdd_expected_fail(context.mock_scenario)
|
|
|
|
|
|
@then('the scenario status should be "{expected}"')
|
|
def step_check_scenario_status(context: Context, expected: str) -> None:
|
|
"""Assert the scenario has the expected status after handler processing."""
|
|
actual = context.mock_scenario.status
|
|
assert actual == _STATUS_MAP[expected], (
|
|
f"Expected scenario status {expected!r}, got {actual!r}"
|
|
)
|
|
|
|
|
|
@then('the step "{step_name}" should have status "{expected}"')
|
|
def step_check_step_status(context: Context, step_name: str, expected: str) -> None:
|
|
"""Assert a named step has the expected status after handler processing."""
|
|
step = context.mock_steps[step_name]
|
|
actual = step.status
|
|
assert actual == _STATUS_MAP[expected], (
|
|
f"Expected step {step_name!r} status {expected!r}, got {actual!r}"
|
|
)
|
|
|
|
|
|
@then('the step "{step_name}" should have error_message cleared')
|
|
def step_check_error_message_cleared(context: Context, step_name: str) -> None:
|
|
"""Assert a named step's ``error_message`` is ``None`` after inversion."""
|
|
step = context.mock_steps[step_name]
|
|
assert step.error_message is None, (
|
|
f"Expected step {step_name!r} error_message to be None after "
|
|
f"inversion, got {step.error_message!r}"
|
|
)
|
|
|
|
|
|
@then('the step "{step_name}" should have a synthetic error_message')
|
|
def step_check_synthetic_error_message(context: Context, step_name: str) -> None:
|
|
"""Assert a named step's ``error_message`` contains the unexpected-pass text."""
|
|
step = context.mock_steps[step_name]
|
|
assert step.error_message is not None, (
|
|
f"Expected step {step_name!r} to have an error_message set, got None"
|
|
)
|
|
assert "Bug appears to be fixed" in step.error_message, (
|
|
f"Expected step {step_name!r} error_message to contain "
|
|
f"'Bug appears to be fixed', got {step.error_message!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _install_tdd_expected_fail_patch tests (S2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("the TDD expected-fail patch installation status is checked")
|
|
def step_check_patch_status(context: Context) -> None:
|
|
"""Verify the patch was installed during ``before_all``."""
|
|
# _install_tdd_expected_fail_patch is called in before_all, which
|
|
# has already run by the time any scenario executes.
|
|
context.patch_flag = getattr(Scenario, "_tdd_run_patched", False)
|
|
|
|
|
|
@then("the Scenario class should have _tdd_run_patched set to True")
|
|
def step_verify_patch_flag(context: Context) -> None:
|
|
"""Assert the patch flag is set on the Scenario class."""
|
|
assert context.patch_flag is True, (
|
|
"Expected Scenario._tdd_run_patched to be True after "
|
|
"_install_tdd_expected_fail_patch(), got False"
|
|
)
|
|
|
|
|
|
@when("_install_tdd_expected_fail_patch is called twice")
|
|
def step_call_patch_twice(context: Context) -> None:
|
|
"""Call ``_install_tdd_expected_fail_patch`` twice and capture ``run``.
|
|
|
|
Saves the current ``Scenario.run`` before the second call so the
|
|
``Then`` step can verify identity. A cleanup handler restores the
|
|
original method in case the idempotency guard is ever broken — this
|
|
prevents a double-wrapped ``Scenario.run`` from leaking into
|
|
subsequent scenarios in the same process.
|
|
"""
|
|
run_before = Scenario.run
|
|
_install_tdd_expected_fail_patch()
|
|
context.run_after_second_call = Scenario.run
|
|
context.run_before_second_call = run_before
|
|
|
|
# TF-1 safety net: restore Scenario.run if the guard failed.
|
|
def _restore_run() -> None:
|
|
Scenario.run = run_before
|
|
|
|
context._cleanup_handlers.append(_restore_run)
|
|
|
|
|
|
@then("the Scenario.run method should not be double-wrapped")
|
|
def step_verify_no_double_wrap(context: Context) -> None:
|
|
"""Assert a second call to the patch does not double-wrap ``Scenario.run``."""
|
|
assert context.run_after_second_call is context.run_before_second_call, (
|
|
"Expected Scenario.run to remain the same after a second call to "
|
|
"_install_tdd_expected_fail_patch() (idempotency), but it was "
|
|
"replaced — indicating double-wrapping"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# before_scenario hook_failed regression test (TC-2)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("before_scenario is called with the mock scenario")
|
|
def step_call_before_scenario(context: Context) -> None:
|
|
"""Invoke ``before_scenario`` with the mock scenario.
|
|
|
|
Uses a lightweight mock ``context`` so the hook's non-TDD setup
|
|
(database paths, container overrides, etc.) runs harmlessly.
|
|
"""
|
|
from unittest.mock import MagicMock
|
|
|
|
mock_context = MagicMock()
|
|
mock_context._cleanup_handlers = []
|
|
mock_context._scenario_db_paths = []
|
|
before_scenario(mock_context, context.mock_scenario)
|
|
|
|
|
|
@then("the scenario hook_failed flag should be True")
|
|
def step_check_hook_failed_true(context: Context) -> None:
|
|
"""Assert the scenario's ``hook_failed`` attribute is ``True``."""
|
|
assert context.mock_scenario.hook_failed is True, (
|
|
"Expected scenario.hook_failed to be True after before_scenario "
|
|
"processes invalid TDD tags, got False"
|
|
)
|