Files
temp/features/steps/tdd_expected_fail_infrastructure_steps.py
Luis Mendes c8cd7eab82 feat(testing): implement @tdd_expected_fail tag handling in Behave environment
Implemented the three-tag TDD bug-capture system in Behave environment hooks:

- Added tag validation in before_scenario: @tdd_bug_<N> requires @tdd_bug,
  @tdd_expected_fail requires both @tdd_bug and @tdd_bug_<N>
- Added result inversion via Scenario.run() wrapper installed in before_all:
  scenarios tagged @tdd_expected_fail that fail are reported as passed
  (expected failure), and scenarios that unexpectedly pass are reported as
  failed with guidance to remove the tag
- Added helper functions validate_tdd_tags() and should_invert_result()
- Added inline documentation referencing CONTRIBUTING.md > TDD Bug Test Tags
- Added Behave test scenarios for tag validation and inversion behavior
- Extract apply_tdd_inversion() as a public, testable function that
  encapsulates all inversion logic with proper guards
- Refactored handle_tdd_expected_fail() to delegate to apply_tdd_inversion()
  after tag validation, eliminating ~55 lines of duplicated inversion logic
- Tag validation errors in handle_tdd_expected_fail() are now logged at
  WARNING level instead of being silently swallowed
- Add hook_failed guard: never invert infrastructure/hook errors
- Add was_dry_run guard: skip inversion when no test actually ran
- Add non-AssertionError guard: warn and skip inversion for exceptions
  that likely indicate infrastructure problems, not the captured bug
- Log exception details at DEBUG level before clearing (previously
  discarded silently)
- Attach synthetic AssertionError to last step on unexpected pass so
  the failure reason appears in standard Behave formatter output

ISSUES CLOSED: #627
2026-03-12 19:49:11 +00:00

288 lines
11 KiB
Python

"""Step definitions for the TDD expected-fail handler infrastructure tests.
These scenarios exercise both ``apply_tdd_inversion`` (the production code
path called by the ``Scenario.run()`` monkey-patch) and
``handle_tdd_expected_fail`` (the standalone entry point) using real Behave
``Scenario`` and ``Step`` objects to verify scenario-level and **step-level**
status inversion, including guard paths (hook errors, dry-run, non-assertion
exceptions).
"""
from __future__ import annotations
from behave import given, then, when
from behave.model import Scenario, Status, Step
from behave.runner import Context
from features.environment import (
_install_tdd_expected_fail_patch,
apply_tdd_inversion,
before_scenario,
handle_tdd_expected_fail,
)
_STATUS_MAP: dict[str, Status] = {
"failed": Status.failed,
"passed": Status.passed,
"skipped": Status.skipped,
"untested": Status.untested,
}
@given('a mock scenario tagged "{tags}" with status "{status}"')
def step_mock_scenario(context: Context, tags: str, status: str) -> None:
"""Build a real ``Scenario`` object with the requested tags and status."""
tag_list = [t.lstrip("@") for t in tags.split()]
scenario = Scenario(
filename="<infrastructure-test>",
line=1,
keyword="Scenario",
name="mock-tdd-scenario",
tags=tag_list,
)
# Force the desired starting status.
scenario.clear_status()
scenario.set_status(_STATUS_MAP[status])
# Initialise guard attributes (Behave sets these during Scenario.run()).
scenario.hook_failed = False
scenario.was_dry_run = False
context.mock_scenario = scenario
context.mock_steps = {}
@given('a mock scenario tagged "{tags}" with status "{status}" and hook_failed')
def step_mock_scenario_hook_failed(context: Context, tags: str, status: str) -> None:
"""Build a real ``Scenario`` with ``hook_failed = True``."""
step_mock_scenario(context, tags, status)
context.mock_scenario.hook_failed = True
@given('a mock scenario tagged "{tags}" with status "{status}" and was_dry_run')
def step_mock_scenario_dry_run(context: Context, tags: str, status: str) -> None:
"""Build a real ``Scenario`` with ``was_dry_run = True``."""
step_mock_scenario(context, tags, status)
context.mock_scenario.was_dry_run = True
@given('the mock scenario has a step "{step_name}" with status "{status}"')
def step_add_mock_step(context: Context, step_name: str, status: str) -> None:
"""Add a real ``Step`` to the mock scenario with the requested status."""
step = Step(
filename="<infrastructure-test>",
line=1,
keyword="Given",
step_type="given",
name=step_name,
)
step.status = _STATUS_MAP[status]
# Provide an error_message for failed steps so logging can use it.
if status == "failed":
step.error_message = f"simulated failure in {step_name!r}"
step.exception = AssertionError(f"simulated failure in {step_name!r}")
context.mock_scenario.steps.append(step)
context.mock_steps[step_name] = step
@given(
'the mock scenario has an infrastructure-error step "{step_name}"'
' with exception "{exc_type}"'
)
def step_add_mock_step_with_exception(
context: Context, step_name: str, exc_type: str
) -> None:
"""Add a real ``Step`` with a specific non-assertion exception type."""
step = Step(
filename="<infrastructure-test>",
line=1,
keyword="Given",
step_type="given",
name=step_name,
)
step.status = Status.failed
exc_classes: dict[str, type[BaseException]] = {
"RuntimeError": RuntimeError,
"TypeError": TypeError,
"ConnectionError": ConnectionError,
"AssertionError": AssertionError,
}
exc_cls = exc_classes.get(exc_type, RuntimeError)
step.exception = exc_cls(f"simulated {exc_type} in {step_name!r}")
step.error_message = f"simulated {exc_type} in {step_name!r}"
context.mock_scenario.steps.append(step)
context.mock_steps[step_name] = step
# ---------------------------------------------------------------------------
# apply_tdd_inversion steps
# ---------------------------------------------------------------------------
@when("apply_tdd_inversion processes the scenario with failed True")
def step_run_apply_inversion_failed(context: Context) -> None:
"""Invoke ``apply_tdd_inversion`` with ``failed=True``."""
context.apply_inversion_result = apply_tdd_inversion(
context.mock_scenario, failed=True
)
@when("apply_tdd_inversion processes the scenario with failed False")
def step_run_apply_inversion_not_failed(context: Context) -> None:
"""Invoke ``apply_tdd_inversion`` with ``failed=False``."""
context.apply_inversion_result = apply_tdd_inversion(
context.mock_scenario, failed=False
)
@then("the apply_tdd_inversion result should be False")
def step_check_inversion_result_false(context: Context) -> None:
"""Assert ``apply_tdd_inversion`` returned ``False``."""
assert context.apply_inversion_result is False, (
f"Expected apply_tdd_inversion to return False, "
f"got {context.apply_inversion_result!r}"
)
@then("the apply_tdd_inversion result should be True")
def step_check_inversion_result_true(context: Context) -> None:
"""Assert ``apply_tdd_inversion`` returned ``True``."""
assert context.apply_inversion_result is True, (
f"Expected apply_tdd_inversion to return True, "
f"got {context.apply_inversion_result!r}"
)
# ---------------------------------------------------------------------------
# handle_tdd_expected_fail steps (standalone entry point)
# ---------------------------------------------------------------------------
@when("the TDD expected-fail handler processes the scenario")
def step_run_handler(context: Context) -> None:
"""Invoke ``handle_tdd_expected_fail`` on the mock scenario."""
handle_tdd_expected_fail(context.mock_scenario)
@then('the scenario status should be "{expected}"')
def step_check_scenario_status(context: Context, expected: str) -> None:
"""Assert the scenario has the expected status after handler processing."""
actual = context.mock_scenario.status
assert actual == _STATUS_MAP[expected], (
f"Expected scenario status {expected!r}, got {actual!r}"
)
@then('the step "{step_name}" should have status "{expected}"')
def step_check_step_status(context: Context, step_name: str, expected: str) -> None:
"""Assert a named step has the expected status after handler processing."""
step = context.mock_steps[step_name]
actual = step.status
assert actual == _STATUS_MAP[expected], (
f"Expected step {step_name!r} status {expected!r}, got {actual!r}"
)
@then('the step "{step_name}" should have error_message cleared')
def step_check_error_message_cleared(context: Context, step_name: str) -> None:
"""Assert a named step's ``error_message`` is ``None`` after inversion."""
step = context.mock_steps[step_name]
assert step.error_message is None, (
f"Expected step {step_name!r} error_message to be None after "
f"inversion, got {step.error_message!r}"
)
@then('the step "{step_name}" should have a synthetic error_message')
def step_check_synthetic_error_message(context: Context, step_name: str) -> None:
"""Assert a named step's ``error_message`` contains the unexpected-pass text."""
step = context.mock_steps[step_name]
assert step.error_message is not None, (
f"Expected step {step_name!r} to have an error_message set, got None"
)
assert "Bug appears to be fixed" in step.error_message, (
f"Expected step {step_name!r} error_message to contain "
f"'Bug appears to be fixed', got {step.error_message!r}"
)
# ---------------------------------------------------------------------------
# _install_tdd_expected_fail_patch tests (S2)
# ---------------------------------------------------------------------------
@when("the TDD expected-fail patch installation status is checked")
def step_check_patch_status(context: Context) -> None:
"""Verify the patch was installed during ``before_all``."""
# _install_tdd_expected_fail_patch is called in before_all, which
# has already run by the time any scenario executes.
context.patch_flag = getattr(Scenario, "_tdd_run_patched", False)
@then("the Scenario class should have _tdd_run_patched set to True")
def step_verify_patch_flag(context: Context) -> None:
"""Assert the patch flag is set on the Scenario class."""
assert context.patch_flag is True, (
"Expected Scenario._tdd_run_patched to be True after "
"_install_tdd_expected_fail_patch(), got False"
)
@when("_install_tdd_expected_fail_patch is called twice")
def step_call_patch_twice(context: Context) -> None:
"""Call ``_install_tdd_expected_fail_patch`` twice and capture ``run``.
Saves the current ``Scenario.run`` before the second call so the
``Then`` step can verify identity. A cleanup handler restores the
original method in case the idempotency guard is ever broken — this
prevents a double-wrapped ``Scenario.run`` from leaking into
subsequent scenarios in the same process.
"""
run_before = Scenario.run
_install_tdd_expected_fail_patch()
context.run_after_second_call = Scenario.run
context.run_before_second_call = run_before
# TF-1 safety net: restore Scenario.run if the guard failed.
def _restore_run() -> None:
Scenario.run = run_before
context._cleanup_handlers.append(_restore_run)
@then("the Scenario.run method should not be double-wrapped")
def step_verify_no_double_wrap(context: Context) -> None:
"""Assert a second call to the patch does not double-wrap ``Scenario.run``."""
assert context.run_after_second_call is context.run_before_second_call, (
"Expected Scenario.run to remain the same after a second call to "
"_install_tdd_expected_fail_patch() (idempotency), but it was "
"replaced — indicating double-wrapping"
)
# ---------------------------------------------------------------------------
# before_scenario hook_failed regression test (TC-2)
# ---------------------------------------------------------------------------
@when("before_scenario is called with the mock scenario")
def step_call_before_scenario(context: Context) -> None:
"""Invoke ``before_scenario`` with the mock scenario.
Uses a lightweight mock ``context`` so the hook's non-TDD setup
(database paths, container overrides, etc.) runs harmlessly.
"""
from unittest.mock import MagicMock
mock_context = MagicMock()
mock_context._cleanup_handlers = []
mock_context._scenario_db_paths = []
before_scenario(mock_context, context.mock_scenario)
@then("the scenario hook_failed flag should be True")
def step_check_hook_failed_true(context: Context) -> None:
"""Assert the scenario's ``hook_failed`` attribute is ``True``."""
assert context.mock_scenario.hook_failed is True, (
"Expected scenario.hook_failed to be True after before_scenario "
"processes invalid TDD tags, got False"
)