From c8cd7eab826ba045ce600f092e922dee0282b8f6 Mon Sep 17 00:00:00 2001 From: Luis Mendes Date: Tue, 10 Mar 2026 00:53:59 +0000 Subject: [PATCH] feat(testing): implement @tdd_expected_fail tag handling in Behave environment Implemented the three-tag TDD bug-capture system in Behave environment hooks: - Added tag validation in before_scenario: @tdd_bug_ requires @tdd_bug, @tdd_expected_fail requires both @tdd_bug and @tdd_bug_ - Added result inversion via Scenario.run() wrapper installed in before_all: scenarios tagged @tdd_expected_fail that fail are reported as passed (expected failure), and scenarios that unexpectedly pass are reported as failed with guidance to remove the tag - Added helper functions validate_tdd_tags() and should_invert_result() - Added inline documentation referencing CONTRIBUTING.md > TDD Bug Test Tags - Added Behave test scenarios for tag validation and inversion behavior - Extract apply_tdd_inversion() as a public, testable function that encapsulates all inversion logic with proper guards - Refactored handle_tdd_expected_fail() to delegate to apply_tdd_inversion() after tag validation, eliminating ~55 lines of duplicated inversion logic - Tag validation errors in handle_tdd_expected_fail() are now logged at WARNING level instead of being silently swallowed - Add hook_failed guard: never invert infrastructure/hook errors - Add was_dry_run guard: skip inversion when no test actually ran - Add non-AssertionError guard: warn and skip inversion for exceptions that likely indicate infrastructure problems, not the captured bug - Log exception details at DEBUG level before clearing (previously discarded silently) - Attach synthetic AssertionError to last step on unexpected pass so the failure reason appears in standard Behave formatter output ISSUES CLOSED: #627 --- CHANGELOG.md | 8 + features/environment.py | 375 +++++++++++++----- .../steps/tdd_expected_fail_demo_steps.py | 39 ++ .../tdd_expected_fail_infrastructure_steps.py | 220 +++++++++- features/steps/tdd_tag_validation_steps.py | 332 ++++++++++++++++ .../tdd_expected_fail_infrastructure.feature | 131 +++++- features/testing/__init__.py | 0 .../testing/tdd_expected_fail_demo.feature | 19 + features/testing/tdd_tag_validation.feature | 126 ++++++ robot/helper_tdd_tag_validation.py | 283 +++++++++++++ robot/tdd_tag_validation.robot | 134 +++++++ 11 files changed, 1563 insertions(+), 104 deletions(-) create mode 100644 features/steps/tdd_expected_fail_demo_steps.py create mode 100644 features/steps/tdd_tag_validation_steps.py create mode 100644 features/testing/__init__.py create mode 100644 features/testing/tdd_expected_fail_demo.feature create mode 100644 features/testing/tdd_tag_validation.feature create mode 100644 robot/helper_tdd_tag_validation.py create mode 100644 robot/tdd_tag_validation.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index ef360dd8c..735677199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,14 @@ protection, and `sync_results_to_host` for file-based result retrieval. Covered by Behave BDD scenarios, Robot Framework integration tests, ASV benchmarks, and `docs/reference/execution_environment.md`. (#515) +- Implemented `@tdd_expected_fail` tag handling in Behave environment hooks. Added + `validate_tdd_tags()` and `should_invert_result()` helper functions in + `features/environment.py`. Scenarios tagged `@tdd_expected_fail` that fail have their + result inverted to pass (expected failure); scenarios that unexpectedly pass are reported + as failed with guidance to remove the tag. Tag validation enforces `@tdd_bug` + + `@tdd_bug_` prerequisites. Implemented via `Scenario.run()` monkey-patch in `before_all`. + Includes 34 Behave BDD scenarios (19 tag-validation, 14 infrastructure, and + 1 demo) and 12 Robot Framework integration test cases. (#627) ### Added diff --git a/features/environment.py b/features/environment.py index b65b880c3..0d45b4107 100644 --- a/features/environment.py +++ b/features/environment.py @@ -10,10 +10,7 @@ import tempfile from pathlib import Path from typing import Any -from behave.model import Scenario -from behave.model_core import Status - -_tdd_logger = logging.getLogger("behave.tdd_expected_fail") +from behave.model import Scenario, Status LANGSMITH_ENV_VARS = [ "CLEVERAGENTS_LANGSMITH_ENABLED", @@ -33,6 +30,264 @@ LANGSMITH_ENV_VARS = [ "LANGSMITH_USER_ID", ] +# --------------------------------------------------------------------------- +# TDD Bug Test Tags — Three-Tag System +# --------------------------------------------------------------------------- +# TDD bug-capture tests use a three-tag system documented in +# CONTRIBUTING.md > Bug Fix Workflow > TDD Bug Test Tags: +# +# @tdd_bug — Generic filter tag. Present on ALL TDD bug tests. +# @tdd_bug_ — Issue reference (e.g. @tdd_bug_123). Links the +# test to the specific Type/Bug issue it captures. +# @tdd_expected_fail — Behavioral switch. When present, the test result +# is inverted: a failure means the bug still exists +# (reported as passed), and a pass means the bug was +# fixed without removing the tag (reported as failed). +# +# The ``validate_tdd_tags`` and ``should_invert_result`` helpers below are +# called from the ``before_scenario`` hook and the ``Scenario.run()`` +# wrapper (installed in ``before_all``) respectively. They are extracted +# as standalone functions so they can be unit-tested directly from Behave +# step definitions. ``apply_tdd_inversion`` encapsulates the full +# inversion logic and is likewise directly testable. +# --------------------------------------------------------------------------- + +_TDD_BUG_N_RE = re.compile(r"tdd_bug_\d+") + +_tdd_logger = logging.getLogger("cleveragents.testing.tdd_tags") + + +def validate_tdd_tags(tags: set[str]) -> None: + """Validate TDD bug-capture tag combinations. + + Raises ``ValueError`` with a descriptive message when the tag set is + inconsistent according to the rules in CONTRIBUTING.md: + + * ``@tdd_bug_`` requires ``@tdd_bug`` to also be present. + * ``@tdd_expected_fail`` requires both ``@tdd_bug`` and at least one + ``@tdd_bug_``. + + Args: + tags: The *effective* tags of a scenario (own tags + feature tags). + + Raises: + ValueError: If the tag combination violates the TDD tag rules. + """ + has_tdd_bug = "tdd_bug" in tags + has_tdd_bug_n = any(_TDD_BUG_N_RE.fullmatch(t) for t in tags) + has_expected_fail = "tdd_expected_fail" in tags + + if has_tdd_bug_n and not has_tdd_bug: + bug_n_tags = sorted(t for t in tags if _TDD_BUG_N_RE.fullmatch(t)) + raise ValueError( + f"Scenario has {', '.join('@' + t for t in bug_n_tags)} but is " + f"missing the required @tdd_bug tag. All TDD bug tests must " + f"include @tdd_bug. See CONTRIBUTING.md > TDD Bug Test Tags." + ) + + if has_expected_fail: + missing: list[str] = [] + if not has_tdd_bug: + missing.append("@tdd_bug") + if not has_tdd_bug_n: + missing.append("@tdd_bug_") + if missing: + raise ValueError( + f"Scenario has @tdd_expected_fail but is missing required " + f"tag(s): {', '.join(missing)}. @tdd_expected_fail requires " + f"both @tdd_bug and at least one @tdd_bug_. " + f"See CONTRIBUTING.md > TDD Bug Test Tags." + ) + + +def should_invert_result(tags: set[str]) -> bool: + """Return ``True`` if the scenario result should be inverted. + + A scenario result is inverted when ``@tdd_expected_fail`` is present in + the effective tags, indicating the test captures a known bug that has not + yet been fixed. + + Args: + tags: The *effective* tags of a scenario (own tags + feature tags). + """ + return "tdd_expected_fail" in tags + + +_UNEXPECTED_PASS_MSG = ( + "Bug appears to be fixed. Remove the @tdd_expected_fail tag " + "from this scenario and verify the fix through the bug fix " + "workflow. See CONTRIBUTING.md > Bug Fix Workflow." +) + + +def apply_tdd_inversion(scenario: Any, failed: bool) -> bool: + """Apply TDD expected-fail result inversion to a completed scenario. + + Encapsulates the full inversion logic so it can be unit-tested directly + from Behave step definitions (using mock scenario objects). + + Guards against infrastructure errors and dry-run mode before inverting: + + * **Hook/cleanup errors** (``scenario.hook_failed``): never inverted — + infrastructure failures must propagate regardless of tags. + * **Dry-run mode** (``scenario.was_dry_run``): never inverted — no test + actually executed, so the result is meaningless. + * **Non-assertion exceptions**: a step that failed with an exception + other than ``AssertionError`` likely indicates an infrastructure + problem, not the captured bug — a warning is logged and the result + is *not* inverted. + + When inverting expected failures, ``step.error_message`` is also cleared + alongside ``step.exception`` and ``step.exc_traceback`` to prevent stale + failure text from leaking to JUnit XML reporters and custom formatters. + + Args: + scenario: A Behave ``Scenario`` (or compatible mock) with at least + ``effective_tags``, ``all_steps``, ``name``, ``hook_failed``, + ``was_dry_run``, ``clear_status()``, and ``set_status()`` + attributes. + failed: The boolean return value of the original ``Scenario.run()``. + + Returns: + The (possibly inverted) failure status to be returned to the runner. + """ + if not should_invert_result(set(scenario.effective_tags)): + return failed + + # Guard: never invert infrastructure / hook errors. + if getattr(scenario, "hook_failed", False): + return failed + + # Guard: no-op during dry-run — no test actually executed. + if getattr(scenario, "was_dry_run", False): + return failed + + # Materialise all_steps once — some Behave versions return an + # iterator (list_iterator / itertools.chain) instead of a list. + all_steps = list(scenario.all_steps) + + if failed: + # Guard: do not invert non-assertion exceptions — they likely + # indicate an infrastructure problem, not the captured bug. + for step in all_steps: + if ( + step.status == Status.failed + and step.exception is not None + and not isinstance(step.exception, AssertionError) + ): + _tdd_logger.warning( + "Non-assertion exception in expected-fail scenario " + "'%s' step '%s': %s — not inverting.", + scenario.name, + step.name, + step.exception, + ) + return failed + + # Expected failure — the bug still exists. Reset the failed + # and skipped steps so the scenario is reported as passed. + # (When a step fails, subsequent steps are marked as skipped; + # both must be set to ``Status.passed`` for accurate summary + # counts and consistent ``compute_status()`` behaviour.) + for step in all_steps: + if step.status in (Status.failed, Status.skipped): + _tdd_logger.debug( + "Clearing expected-fail exception for step '%s': %s", + step.name, + step.exception, + ) + step.status = Status.passed + step.exception = None + step.exc_traceback = None + step.error_message = None + scenario.clear_status() + scenario.set_status(Status.passed) + return False # Not a failure for the runner + + # Unexpected pass — the bug appears to be fixed but the + # @tdd_expected_fail tag has not been removed. Force a failure + # so CI blocks the PR until the tag is cleaned up. + _tdd_logger.warning( + "Bug appears to be fixed. Remove the @tdd_expected_fail " + "tag from scenario '%s' and verify the fix through the " + "bug fix workflow. See CONTRIBUTING.md > Bug Fix Workflow.", + scenario.name, + ) + # Attach a synthetic error to the last step so the failure reason + # appears in standard Behave output (formatters show the failed + # step's exception text). + if all_steps: + last_step = all_steps[-1] + last_step.status = Status.failed + last_step.exception = AssertionError(_UNEXPECTED_PASS_MSG) + last_step.error_message = "Assertion Failed: " + _UNEXPECTED_PASS_MSG + scenario.set_status(Status.failed) + return True # Force failure for the runner + + +def handle_tdd_expected_fail(scenario: Any) -> None: + """Process a scenario through TDD expected-fail tag validation and inversion. + + Public, testable entry point that encapsulates the full TDD expected-fail + logic operating directly on a ``Scenario`` object: + + 1. **Tag validation** — rejects invalid TDD tag combinations by forcing + the scenario to ``Status.failed``. + 2. **Result inversion** — delegates to :func:`apply_tdd_inversion` for + valid ``@tdd_expected_fail`` scenarios, which inverts the result: + a failure becomes passed (expected), a pass becomes failed + (unexpected — the bug appears fixed). + + All guard logic (hook errors, dry-run, non-assertion exceptions) and + step-level status manipulation are handled by :func:`apply_tdd_inversion`. + + Args: + scenario: A Behave ``Scenario`` (or compatible mock) with at least + ``effective_tags`` (or ``tags``), ``all_steps``, ``status``, + ``hook_failed``, ``was_dry_run``, ``clear_status()``, and + ``set_status()``. + """ + tags = set(getattr(scenario, "effective_tags", getattr(scenario, "tags", []))) + + # Validate tags — force failure on invalid combinations. + try: + validate_tdd_tags(tags) + except ValueError as exc: + _tdd_logger.warning( + "Invalid TDD tag combination on scenario '%s': %s", + scenario.name, + exc, + ) + scenario.set_status(Status.failed) + return + + # Delegate inversion logic to the shared implementation. + apply_tdd_inversion(scenario, scenario.status == Status.failed) + + +def _install_tdd_expected_fail_patch() -> None: + """Monkey-patch ``Scenario.run`` to invert results for ``@tdd_expected_fail``. + + Behave's ``Scenario.run()`` returns a local ``failed`` boolean that + ``after_scenario`` hooks cannot modify. To correctly invert the result + (so that an expected-fail scenario is reported as passed to the runner), + we wrap ``Scenario.run()`` with a thin post-processing layer that + delegates to :func:`apply_tdd_inversion`. + + The patch is installed once in ``before_all`` and is idempotent. + """ + if getattr(Scenario, "_tdd_run_patched", False): + return # Already patched (e.g. forked worker reloading hooks) + + _original_run = Scenario.run + + def _tdd_aware_run(self: Any, runner: Any) -> bool: + failed: bool = _original_run(self, runner) + return apply_tdd_inversion(self, failed) + + Scenario.run = _tdd_aware_run + Scenario._tdd_run_patched = True + def before_all(context): """Set up test environment before all tests.""" @@ -85,6 +340,11 @@ def before_all(context): _ensure_template_db() _install_template_db_patch() + # --- TDD Expected-Fail Patch --- + # Wrap Scenario.run() so @tdd_expected_fail inverts the result for + # both the scenario status AND the runner's pass/fail return value. + _install_tdd_expected_fail_patch() + def _install_fast_sleep_patch() -> None: """Cap ``time.sleep`` and ``asyncio.sleep`` at 10 ms for fast test execution. @@ -230,6 +490,18 @@ def _install_template_db_patch() -> None: def before_scenario(context, scenario): """Set up before each scenario.""" + # --- TDD Bug Tag Validation --- + # Validate the three-tag system BEFORE any other setup so that + # misconfigured TDD tests are caught immediately. + # See CONTRIBUTING.md > TDD Bug Test Tags for the full specification. + try: + validate_tdd_tags(set(scenario.effective_tags)) + except ValueError as exc: + scenario.hook_failed = True + scenario.set_status(Status.failed) + _tdd_logger.error("TDD TAG ERROR in %r: %s", scenario.name, exc) + return + # Store original working directory context.original_cwd = os.getcwd() @@ -317,96 +589,8 @@ def before_scenario(context, scenario): pass # Container not needed for all tests -# --------------------------------------------------------------------------- -# TDD expected-fail tag handler (see CONTRIBUTING.md §TDD Bug Test Tags) -# --------------------------------------------------------------------------- - -_TDD_BUG_N_RE = re.compile(r"^tdd_bug_\d+$") - - -def handle_tdd_expected_fail(scenario: Scenario) -> None: - """Validate TDD tags and invert pass/fail for ``@tdd_expected_fail``. - - Tag validation (per CONTRIBUTING.md — unconditional): - - * Any scenario with ``@tdd_bug_`` **must** also carry ``@tdd_bug``. - Missing ``@tdd_bug`` causes the scenario to fail unconditionally. - - Status inversion (only when ``@tdd_expected_fail`` is present): - - * ``@tdd_expected_fail`` additionally requires ``@tdd_bug`` **and** at - least one ``@tdd_bug_`` tag. - * **failed → passed** — the bug still triggers, which is expected. - * **passed → failed** — the bug was fixed but the tag was not removed; - this is an error that must be caught. - """ - tags = set(scenario.effective_tags) - - # --- unconditional tag validation (F5) -------------------------------- - has_tdd_bug_n = any(_TDD_BUG_N_RE.match(t) for t in tags) - if has_tdd_bug_n and "tdd_bug" not in tags: - scenario.set_status(Status.failed) - sys.stderr.write( - f"TDD TAG ERROR: {scenario.name!r} — @tdd_bug_ requires @tdd_bug tag\n" - ) - return - - if "tdd_expected_fail" not in tags: - return - - # --- @tdd_expected_fail tag validation -------------------------------- - if "tdd_bug" not in tags: - scenario.set_status(Status.failed) - sys.stderr.write( - f"TDD TAG ERROR: {scenario.name!r} — " - "@tdd_expected_fail requires @tdd_bug tag\n" - ) - return - - if not has_tdd_bug_n: - scenario.set_status(Status.failed) - sys.stderr.write( - f"TDD TAG ERROR: {scenario.name!r} — " - "@tdd_expected_fail requires at least one @tdd_bug_ tag\n" - ) - return - - # --- status inversion ------------------------------------------------- - if scenario.status == Status.failed: - # Log original failure details before inverting so CI logs show what - # actually failed (N3 review finding). - for step in scenario.steps: - if step.status == Status.failed: - _tdd_logger.info( - "TDD inversion: %s — step %r failed: %s", - scenario.name, - step.name, - step.error_message or "(no message)", - ) - - # Bug still present — expected. Mark scenario and its failed/skipped - # steps as passed so that summary counts are accurate. - scenario.clear_status() - scenario.set_status(Status.passed) - for step in scenario.steps: - if step.status in (Status.failed, Status.skipped): - step.status = Status.passed - elif scenario.status == Status.passed: - # Bug was fixed but @tdd_expected_fail was not removed — error. - scenario.set_status(Status.failed) - sys.stderr.write( - f"TDD TAG ERROR: {scenario.name!r} — " - "scenario passed but still carries @tdd_expected_fail; " - "remove the tag now that the bug is fixed\n" - ) - - def after_scenario(context, scenario): """Clean up after each scenario.""" - # Handle TDD expected-fail inversion BEFORE cleanup (status is already set - # by step execution; cleanup does not change it). - handle_tdd_expected_fail(scenario) - # Return to original directory first if hasattr(context, "original_cwd"): os.chdir(context.original_cwd) @@ -511,8 +695,6 @@ def after_scenario(context, scenario): # T6: Remove log handlers attached to the async-cleanup logger by # security_async_steps.py so handlers don't accumulate across scenarios. - import logging - if hasattr(context, "log_handler"): async_logger = logging.getLogger("cleveragents.core.async_cleanup") async_logger.removeHandler(context.log_handler) @@ -521,3 +703,8 @@ def after_scenario(context, scenario): if hasattr(context, "bridge_loop"): with contextlib.suppress(Exception): context.bridge_loop.close() + + # NOTE: TDD @tdd_expected_fail result inversion is handled by the + # Scenario.run() wrapper installed in _install_tdd_expected_fail_patch(), + # NOT in this hook. See before_all() and CONTRIBUTING.md > TDD Bug + # Test Tags for the full specification. diff --git a/features/steps/tdd_expected_fail_demo_steps.py b/features/steps/tdd_expected_fail_demo_steps.py new file mode 100644 index 000000000..bca337032 --- /dev/null +++ b/features/steps/tdd_expected_fail_demo_steps.py @@ -0,0 +1,39 @@ +"""Step definitions used exclusively by ``tdd_expected_fail_demo.feature``. + +Per CONTRIBUTING.md: steps used only by a particular feature file must live +in a correspondingly named step definition file. + +These steps provide deliberately-passing and deliberately-failing behaviour +to exercise the ``Scenario.run()`` wrapper installed by +``_install_tdd_expected_fail_patch()`` in ``features/environment.py``. + +See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification. +""" + +from __future__ import annotations + +from behave import given, then, when +from behave.runner import Context + + +@given("tdd demo a step that always succeeds") +def step_tdd_demo_success(context: Context) -> None: + """A trivial passing step.""" + context.tdd_demo_ran = True + + +@when("tdd demo a deliberately failing assertion is executed") +def step_tdd_demo_deliberate_fail(context: Context) -> None: + """This step deliberately fails to simulate a bug still being present. + + The ``@tdd_expected_fail`` tag on the scenario causes the + ``Scenario.run()`` wrapper to invert this failure into a pass. + """ + msg = "Deliberate failure: bug #999 is still present (expected by TDD tag)" + raise AssertionError(msg) + + +@then("tdd demo this step is never reached") +def step_tdd_demo_never_reached(context: Context) -> None: + """Placeholder step — unreachable because the previous step fails.""" + context.tdd_demo_unreachable = True diff --git a/features/steps/tdd_expected_fail_infrastructure_steps.py b/features/steps/tdd_expected_fail_infrastructure_steps.py index 8bbf6df62..141fa3d9a 100644 --- a/features/steps/tdd_expected_fail_infrastructure_steps.py +++ b/features/steps/tdd_expected_fail_infrastructure_steps.py @@ -1,18 +1,25 @@ """Step definitions for the TDD expected-fail handler infrastructure tests. -These scenarios exercise ``_handle_tdd_expected_fail`` directly using real -Behave ``Scenario`` and ``Step`` objects (not the running test's own scenario) -to verify both scenario-level and **step-level** status inversion. +These scenarios exercise both ``apply_tdd_inversion`` (the production code +path called by the ``Scenario.run()`` monkey-patch) and +``handle_tdd_expected_fail`` (the standalone entry point) using real Behave +``Scenario`` and ``Step`` objects to verify scenario-level and **step-level** +status inversion, including guard paths (hook errors, dry-run, non-assertion +exceptions). """ from __future__ import annotations from behave import given, then, when -from behave.model import Scenario, Step -from behave.model_core import Status +from behave.model import Scenario, Status, Step from behave.runner import Context -from features.environment import handle_tdd_expected_fail +from features.environment import ( + _install_tdd_expected_fail_patch, + apply_tdd_inversion, + before_scenario, + handle_tdd_expected_fail, +) _STATUS_MAP: dict[str, Status] = { "failed": Status.failed, @@ -36,10 +43,27 @@ def step_mock_scenario(context: Context, tags: str, status: str) -> None: # Force the desired starting status. scenario.clear_status() scenario.set_status(_STATUS_MAP[status]) + # Initialise guard attributes (Behave sets these during Scenario.run()). + scenario.hook_failed = False + scenario.was_dry_run = False context.mock_scenario = scenario context.mock_steps = {} +@given('a mock scenario tagged "{tags}" with status "{status}" and hook_failed') +def step_mock_scenario_hook_failed(context: Context, tags: str, status: str) -> None: + """Build a real ``Scenario`` with ``hook_failed = True``.""" + step_mock_scenario(context, tags, status) + context.mock_scenario.hook_failed = True + + +@given('a mock scenario tagged "{tags}" with status "{status}" and was_dry_run') +def step_mock_scenario_dry_run(context: Context, tags: str, status: str) -> None: + """Build a real ``Scenario`` with ``was_dry_run = True``.""" + step_mock_scenario(context, tags, status) + context.mock_scenario.was_dry_run = True + + @given('the mock scenario has a step "{step_name}" with status "{status}"') def step_add_mock_step(context: Context, step_name: str, status: str) -> None: """Add a real ``Step`` to the mock scenario with the requested status.""" @@ -51,16 +75,90 @@ def step_add_mock_step(context: Context, step_name: str, status: str) -> None: name=step_name, ) step.status = _STATUS_MAP[status] - # Provide an error_message for failed steps so logging (N3) can use it. + # Provide an error_message for failed steps so logging can use it. if status == "failed": step.error_message = f"simulated failure in {step_name!r}" + step.exception = AssertionError(f"simulated failure in {step_name!r}") context.mock_scenario.steps.append(step) context.mock_steps[step_name] = step +@given( + 'the mock scenario has an infrastructure-error step "{step_name}"' + ' with exception "{exc_type}"' +) +def step_add_mock_step_with_exception( + context: Context, step_name: str, exc_type: str +) -> None: + """Add a real ``Step`` with a specific non-assertion exception type.""" + step = Step( + filename="", + line=1, + keyword="Given", + step_type="given", + name=step_name, + ) + step.status = Status.failed + exc_classes: dict[str, type[BaseException]] = { + "RuntimeError": RuntimeError, + "TypeError": TypeError, + "ConnectionError": ConnectionError, + "AssertionError": AssertionError, + } + exc_cls = exc_classes.get(exc_type, RuntimeError) + step.exception = exc_cls(f"simulated {exc_type} in {step_name!r}") + step.error_message = f"simulated {exc_type} in {step_name!r}" + context.mock_scenario.steps.append(step) + context.mock_steps[step_name] = step + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion steps +# --------------------------------------------------------------------------- + + +@when("apply_tdd_inversion processes the scenario with failed True") +def step_run_apply_inversion_failed(context: Context) -> None: + """Invoke ``apply_tdd_inversion`` with ``failed=True``.""" + context.apply_inversion_result = apply_tdd_inversion( + context.mock_scenario, failed=True + ) + + +@when("apply_tdd_inversion processes the scenario with failed False") +def step_run_apply_inversion_not_failed(context: Context) -> None: + """Invoke ``apply_tdd_inversion`` with ``failed=False``.""" + context.apply_inversion_result = apply_tdd_inversion( + context.mock_scenario, failed=False + ) + + +@then("the apply_tdd_inversion result should be False") +def step_check_inversion_result_false(context: Context) -> None: + """Assert ``apply_tdd_inversion`` returned ``False``.""" + assert context.apply_inversion_result is False, ( + f"Expected apply_tdd_inversion to return False, " + f"got {context.apply_inversion_result!r}" + ) + + +@then("the apply_tdd_inversion result should be True") +def step_check_inversion_result_true(context: Context) -> None: + """Assert ``apply_tdd_inversion`` returned ``True``.""" + assert context.apply_inversion_result is True, ( + f"Expected apply_tdd_inversion to return True, " + f"got {context.apply_inversion_result!r}" + ) + + +# --------------------------------------------------------------------------- +# handle_tdd_expected_fail steps (standalone entry point) +# --------------------------------------------------------------------------- + + @when("the TDD expected-fail handler processes the scenario") def step_run_handler(context: Context) -> None: - """Invoke ``_handle_tdd_expected_fail`` on the mock scenario.""" + """Invoke ``handle_tdd_expected_fail`` on the mock scenario.""" handle_tdd_expected_fail(context.mock_scenario) @@ -81,3 +179,109 @@ def step_check_step_status(context: Context, step_name: str, expected: str) -> N assert actual == _STATUS_MAP[expected], ( f"Expected step {step_name!r} status {expected!r}, got {actual!r}" ) + + +@then('the step "{step_name}" should have error_message cleared') +def step_check_error_message_cleared(context: Context, step_name: str) -> None: + """Assert a named step's ``error_message`` is ``None`` after inversion.""" + step = context.mock_steps[step_name] + assert step.error_message is None, ( + f"Expected step {step_name!r} error_message to be None after " + f"inversion, got {step.error_message!r}" + ) + + +@then('the step "{step_name}" should have a synthetic error_message') +def step_check_synthetic_error_message(context: Context, step_name: str) -> None: + """Assert a named step's ``error_message`` contains the unexpected-pass text.""" + step = context.mock_steps[step_name] + assert step.error_message is not None, ( + f"Expected step {step_name!r} to have an error_message set, got None" + ) + assert "Bug appears to be fixed" in step.error_message, ( + f"Expected step {step_name!r} error_message to contain " + f"'Bug appears to be fixed', got {step.error_message!r}" + ) + + +# --------------------------------------------------------------------------- +# _install_tdd_expected_fail_patch tests (S2) +# --------------------------------------------------------------------------- + + +@when("the TDD expected-fail patch installation status is checked") +def step_check_patch_status(context: Context) -> None: + """Verify the patch was installed during ``before_all``.""" + # _install_tdd_expected_fail_patch is called in before_all, which + # has already run by the time any scenario executes. + context.patch_flag = getattr(Scenario, "_tdd_run_patched", False) + + +@then("the Scenario class should have _tdd_run_patched set to True") +def step_verify_patch_flag(context: Context) -> None: + """Assert the patch flag is set on the Scenario class.""" + assert context.patch_flag is True, ( + "Expected Scenario._tdd_run_patched to be True after " + "_install_tdd_expected_fail_patch(), got False" + ) + + +@when("_install_tdd_expected_fail_patch is called twice") +def step_call_patch_twice(context: Context) -> None: + """Call ``_install_tdd_expected_fail_patch`` twice and capture ``run``. + + Saves the current ``Scenario.run`` before the second call so the + ``Then`` step can verify identity. A cleanup handler restores the + original method in case the idempotency guard is ever broken — this + prevents a double-wrapped ``Scenario.run`` from leaking into + subsequent scenarios in the same process. + """ + run_before = Scenario.run + _install_tdd_expected_fail_patch() + context.run_after_second_call = Scenario.run + context.run_before_second_call = run_before + + # TF-1 safety net: restore Scenario.run if the guard failed. + def _restore_run() -> None: + Scenario.run = run_before + + context._cleanup_handlers.append(_restore_run) + + +@then("the Scenario.run method should not be double-wrapped") +def step_verify_no_double_wrap(context: Context) -> None: + """Assert a second call to the patch does not double-wrap ``Scenario.run``.""" + assert context.run_after_second_call is context.run_before_second_call, ( + "Expected Scenario.run to remain the same after a second call to " + "_install_tdd_expected_fail_patch() (idempotency), but it was " + "replaced — indicating double-wrapping" + ) + + +# --------------------------------------------------------------------------- +# before_scenario hook_failed regression test (TC-2) +# --------------------------------------------------------------------------- + + +@when("before_scenario is called with the mock scenario") +def step_call_before_scenario(context: Context) -> None: + """Invoke ``before_scenario`` with the mock scenario. + + Uses a lightweight mock ``context`` so the hook's non-TDD setup + (database paths, container overrides, etc.) runs harmlessly. + """ + from unittest.mock import MagicMock + + mock_context = MagicMock() + mock_context._cleanup_handlers = [] + mock_context._scenario_db_paths = [] + before_scenario(mock_context, context.mock_scenario) + + +@then("the scenario hook_failed flag should be True") +def step_check_hook_failed_true(context: Context) -> None: + """Assert the scenario's ``hook_failed`` attribute is ``True``.""" + assert context.mock_scenario.hook_failed is True, ( + "Expected scenario.hook_failed to be True after before_scenario " + "processes invalid TDD tags, got False" + ) diff --git a/features/steps/tdd_tag_validation_steps.py b/features/steps/tdd_tag_validation_steps.py new file mode 100644 index 000000000..d9f5f4648 --- /dev/null +++ b/features/steps/tdd_tag_validation_steps.py @@ -0,0 +1,332 @@ +"""Step definitions for TDD bug-capture tag validation scenarios. + +Tests the ``validate_tdd_tags()``, ``should_invert_result()``, and +``apply_tdd_inversion()`` helper functions defined in +``features/environment.py``. + +All step names are prefixed with ``tdd tags`` or ``tdd inversion`` to avoid +``AmbiguousStep`` conflicts with existing steps. + +See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from behave import given, then, when +from behave.model import Status +from behave.runner import Context + +from features.environment import ( + apply_tdd_inversion, + should_invert_result, + validate_tdd_tags, +) + +# --------------------------------------------------------------------------- +# Tag-set construction helpers +# --------------------------------------------------------------------------- + + +@given('tdd tags a tag set with "{tags_csv}"') +def step_given_tag_set(context: Context, tags_csv: str) -> None: + """Parse a comma-separated tag list into a set on the context.""" + context.tdd_tag_set = {t.strip() for t in tags_csv.split(",") if t.strip()} + + +@given("tdd tags an empty tag set") +def step_given_empty_tag_set(context: Context) -> None: + context.tdd_tag_set = set() + + +# --------------------------------------------------------------------------- +# Validation execution +# --------------------------------------------------------------------------- + + +@when("tdd tags I validate the tag set") +def step_when_validate(context: Context) -> None: + """Run ``validate_tdd_tags`` and capture any raised ``ValueError``.""" + context.tdd_validation_error = None + try: + validate_tdd_tags(context.tdd_tag_set) + except ValueError as exc: + context.tdd_validation_error = str(exc) + + +# --------------------------------------------------------------------------- +# Validation outcome assertions +# --------------------------------------------------------------------------- + + +@then("tdd tags validation should pass") +def step_then_validation_passes(context: Context) -> None: + assert context.tdd_validation_error is None, ( + f"Expected validation to pass but got error: {context.tdd_validation_error}" + ) + + +@then('tdd tags validation should fail with error containing "{fragment}"') +def step_then_validation_fails_with(context: Context, fragment: str) -> None: + assert context.tdd_validation_error is not None, ( + "Expected validation to fail but it passed" + ) + assert fragment in context.tdd_validation_error, ( + f"Expected error to contain '{fragment}' but got: " + f"{context.tdd_validation_error}" + ) + + +# --------------------------------------------------------------------------- +# should_invert_result assertions +# --------------------------------------------------------------------------- + + +@then("tdd tags should_invert_result should return true") +def step_then_invert_true(context: Context) -> None: + assert should_invert_result(context.tdd_tag_set) is True + + +@then("tdd tags should_invert_result should return false") +def step_then_invert_false(context: Context) -> None: + assert should_invert_result(context.tdd_tag_set) is False + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion unit-test helpers +# --------------------------------------------------------------------------- + + +def _make_mock_scenario( + tags: list[str], + steps_passed: bool = True, + hook_failed: bool = False, + was_dry_run: bool = False, + step_exception: BaseException | None = None, +) -> MagicMock: + """Build a lightweight mock ``Scenario`` for ``apply_tdd_inversion`` tests. + + Args: + tags: The effective tags to place on the scenario. + steps_passed: When ``True`` every step has ``Status.passed``. + When ``False`` the first step has ``Status.failed`` with + *step_exception* (defaulting to ``AssertionError``). + hook_failed: Simulates an infrastructure/hook error. + was_dry_run: Simulates ``--dry-run`` mode. + step_exception: The exception attached to the failed step when + *steps_passed* is ``False``. Defaults to ``AssertionError``. + """ + scenario = MagicMock() + scenario.effective_tags = tags + scenario.name = "mock-scenario" + scenario.hook_failed = hook_failed + scenario.was_dry_run = was_dry_run + + mock_step = MagicMock() + if steps_passed: + mock_step.status = Status.passed + mock_step.exception = None + else: + mock_step.status = Status.failed + mock_step.exception = ( + step_exception + if step_exception is not None + else AssertionError("simulated assertion failure") + ) + scenario.all_steps = [mock_step] + return scenario + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — expected failure inverted to pass (happy path) +# --------------------------------------------------------------------------- + + +@given( + "tdd inversion a mock scenario that fails with AssertionError" + " and tdd_expected_fail tags" +) +def step_given_expected_failure_scenario(context: Context) -> None: + """Create a mock scenario with a failed step (AssertionError) and TDD tags.""" + context.tdd_mock_scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"], + steps_passed=False, + step_exception=AssertionError("simulated bug assertion failure"), + ) + + +@then( + "tdd inversion the result should be False indicating expected failure was inverted" +) +def step_then_result_inverted_to_pass(context: Context) -> None: + assert context.tdd_inversion_result is False, ( + f"Expected False (failure inverted to pass) but got " + f"{context.tdd_inversion_result}" + ) + + +@then("tdd inversion the scenario status should be passed") +def step_then_scenario_status_passed(context: Context) -> None: + context.tdd_mock_scenario.set_status.assert_called_with(Status.passed) + + +@then("tdd inversion all failed and skipped steps should be reset to passed") +def step_then_steps_reset_to_passed(context: Context) -> None: + for step in context.tdd_mock_scenario.all_steps: + assert step.status == Status.passed, ( + f"Expected step status to be passed but got {step.status}" + ) + assert step.exception is None, ( + f"Expected step exception to be None but got {step.exception}" + ) + assert step.exc_traceback is None, ( + f"Expected step exc_traceback to be None but got {step.exc_traceback}" + ) + + +@then("tdd inversion all step error_messages should be cleared") +def step_then_error_messages_cleared(context: Context) -> None: + """Assert every step's ``error_message`` is ``None`` after inversion.""" + for step in context.tdd_mock_scenario.all_steps: + assert step.error_message is None, ( + f"Expected step error_message to be None but got {step.error_message!r}" + ) + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — unexpected-pass path +# --------------------------------------------------------------------------- + + +@given("tdd inversion a mock scenario that passes with tdd_expected_fail tags") +def step_given_unexpected_pass_scenario(context: Context) -> None: + """Create a mock scenario where steps pass but @tdd_expected_fail is set.""" + context.tdd_mock_scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"], + steps_passed=True, + ) + + +@when("tdd inversion apply_tdd_inversion is called with failed False") +def step_when_apply_inversion_not_failed(context: Context) -> None: + context.tdd_inversion_result = apply_tdd_inversion( + context.tdd_mock_scenario, failed=False + ) + + +@then("tdd inversion the result should be True indicating forced failure") +def step_then_result_is_forced_failure(context: Context) -> None: + assert context.tdd_inversion_result is True, ( + f"Expected True (forced failure) but got {context.tdd_inversion_result}" + ) + + +@then("tdd inversion the scenario status should be failed") +def step_then_scenario_status_failed(context: Context) -> None: + context.tdd_mock_scenario.set_status.assert_called_with(Status.failed) + + +@then("tdd inversion the last step should have a synthetic error message") +def step_then_last_step_has_error(context: Context) -> None: + last_step = context.tdd_mock_scenario.all_steps[-1] + assert last_step.status == Status.failed, ( + f"Expected last step status to be failed but got {last_step.status}" + ) + assert isinstance(last_step.exception, AssertionError), ( + f"Expected AssertionError but got {type(last_step.exception)}" + ) + assert "Bug appears to be fixed" in str(last_step.exception) + + +@then("tdd inversion the last step should have a synthetic error_message text") +def step_then_last_step_has_error_message(context: Context) -> None: + """Assert the last step's ``error_message`` contains the unexpected-pass text.""" + last_step = context.tdd_mock_scenario.all_steps[-1] + assert last_step.error_message is not None, ( + "Expected last step to have an error_message set, got None" + ) + assert "Bug appears to be fixed" in last_step.error_message, ( + f"Expected error_message to contain 'Bug appears to be fixed', " + f"got {last_step.error_message!r}" + ) + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — hook-error guard +# --------------------------------------------------------------------------- + + +@given("tdd inversion a mock scenario with hook_failed and tdd_expected_fail tags") +def step_given_hook_failed_scenario(context: Context) -> None: + context.tdd_mock_scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"], + steps_passed=False, + hook_failed=True, + ) + + +@when("tdd inversion apply_tdd_inversion is called with failed True") +def step_when_apply_inversion_failed(context: Context) -> None: + context.tdd_inversion_result = apply_tdd_inversion( + context.tdd_mock_scenario, failed=True + ) + + +@then("tdd inversion the result should be True indicating failure was not inverted") +def step_then_result_not_inverted(context: Context) -> None: + assert context.tdd_inversion_result is True, ( + f"Expected True (not inverted) but got {context.tdd_inversion_result}" + ) + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — dry-run guard +# --------------------------------------------------------------------------- + + +@given("tdd inversion a mock scenario in dry-run mode with tdd_expected_fail tags") +def step_given_dry_run_scenario(context: Context) -> None: + context.tdd_mock_scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"], + steps_passed=True, + was_dry_run=True, + ) + + +@then("tdd inversion the result should be False indicating no inversion occurred") +def step_then_result_no_inversion(context: Context) -> None: + assert context.tdd_inversion_result is False, ( + f"Expected False (no inversion) but got {context.tdd_inversion_result}" + ) + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — non-assertion exception guard +# --------------------------------------------------------------------------- + + +@given("tdd inversion a mock scenario with a RuntimeError and tdd_expected_fail tags") +def step_given_non_assertion_scenario(context: Context) -> None: + context.tdd_mock_scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"], + steps_passed=False, + step_exception=RuntimeError("connection lost"), + ) + + +# --------------------------------------------------------------------------- +# apply_tdd_inversion — empty steps edge case +# --------------------------------------------------------------------------- + + +@given("tdd inversion a mock scenario with no steps and tdd_expected_fail tags") +def step_given_no_steps_scenario(context: Context) -> None: + """Create a mock scenario with no steps and @tdd_expected_fail tags.""" + scenario = MagicMock() + scenario.effective_tags = ["tdd_bug", "tdd_bug_998", "tdd_expected_fail"] + scenario.name = "mock-scenario-no-steps" + scenario.hook_failed = False + scenario.was_dry_run = False + scenario.all_steps = [] + context.tdd_mock_scenario = scenario diff --git a/features/tdd_expected_fail_infrastructure.feature b/features/tdd_expected_fail_infrastructure.feature index 9729104dd..ef2c82380 100644 --- a/features/tdd_expected_fail_infrastructure.feature +++ b/features/tdd_expected_fail_infrastructure.feature @@ -1,7 +1,57 @@ @infrastructure Feature: TDD expected-fail handler infrastructure - Verify that the ``_handle_tdd_expected_fail`` hook in ``environment.py`` - correctly inverts scenario AND step-level status for TDD bug-capture tests. + Verify that ``apply_tdd_inversion`` (the production code path called by the + ``Scenario.run()`` monkey-patch) and ``handle_tdd_expected_fail`` (the + standalone entry point) correctly invert scenario AND step-level status for + TDD bug-capture tests. + + # --- apply_tdd_inversion (production path) --- + + Scenario: apply_tdd_inversion inverts a failed scenario with failed and skipped steps to passed + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "broken step" with status "failed" + And the mock scenario has a step "skipped step" with status "skipped" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "passed" + And the step "broken step" should have status "passed" + And the step "skipped step" should have status "passed" + And the step "broken step" should have error_message cleared + And the step "skipped step" should have error_message cleared + + Scenario: apply_tdd_inversion fails a passed scenario that still carries @tdd_expected_fail + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed" + And the mock scenario has a step "last step" with status "passed" + When apply_tdd_inversion processes the scenario with failed False + Then the scenario status should be "failed" + And the step "last step" should have a synthetic error_message + + Scenario: apply_tdd_inversion does not invert when hook_failed is set + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" and hook_failed + And the mock scenario has a step "broken step" with status "failed" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "failed" + And the step "broken step" should have status "failed" + + Scenario: apply_tdd_inversion does not invert during dry-run mode + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed" and was_dry_run + When apply_tdd_inversion processes the scenario with failed False + Then the apply_tdd_inversion result should be False + + Scenario: apply_tdd_inversion does not invert non-AssertionError exceptions + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "failed" + And the step "infra step" should have status "failed" + + Scenario: apply_tdd_inversion ignores scenarios without @tdd_expected_fail + Given a mock scenario tagged "@tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "broken step" with status "failed" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "failed" + And the step "broken step" should have status "failed" + + # --- handle_tdd_expected_fail (standalone entry point) --- Scenario: Handler inverts a failed scenario with failed and skipped steps to passed Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" @@ -11,11 +61,15 @@ Feature: TDD expected-fail handler infrastructure Then the scenario status should be "passed" And the step "broken step" should have status "passed" And the step "skipped step" should have status "passed" + And the step "broken step" should have error_message cleared + And the step "skipped step" should have error_message cleared Scenario: Handler fails a passed scenario that still carries @tdd_expected_fail Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed" + And the mock scenario has a step "last step" with status "passed" When the TDD expected-fail handler processes the scenario Then the scenario status should be "failed" + And the step "last step" should have a synthetic error_message Scenario: Handler rejects @tdd_expected_fail without @tdd_bug Given a mock scenario tagged "@tdd_expected_fail" with status "failed" @@ -36,3 +90,76 @@ Feature: TDD expected-fail handler infrastructure Given a mock scenario tagged "@tdd_bug @tdd_bug_999" with status "failed" When the TDD expected-fail handler processes the scenario Then the scenario status should be "failed" + + Scenario: Handler does not invert non-AssertionError exceptions + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError" + When the TDD expected-fail handler processes the scenario + Then the scenario status should be "failed" + And the step "infra step" should have status "failed" + + # --- apply_tdd_inversion: mixed exception types (S3) --- + + Scenario: apply_tdd_inversion does not invert when mixed exceptions present + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "bug step" with status "failed" + And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "failed" + And the step "bug step" should have status "failed" + And the step "infra step" should have status "failed" + + Scenario: Handler does not invert when mixed exceptions present + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "bug step" with status "failed" + And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError" + When the TDD expected-fail handler processes the scenario + Then the scenario status should be "failed" + And the step "bug step" should have status "failed" + And the step "infra step" should have status "failed" + + # --- selective step reset preserves already-passed steps (TC-3) --- + + Scenario: apply_tdd_inversion preserves already-passed steps during inversion + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "ok step" with status "passed" + And the mock scenario has a step "broken step" with status "failed" + And the mock scenario has a step "skipped step" with status "skipped" + When apply_tdd_inversion processes the scenario with failed True + Then the scenario status should be "passed" + And the step "ok step" should have status "passed" + And the step "broken step" should have status "passed" + And the step "skipped step" should have status "passed" + And the step "broken step" should have error_message cleared + And the step "skipped step" should have error_message cleared + + Scenario: Handler preserves already-passed steps during inversion + Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" + And the mock scenario has a step "ok step" with status "passed" + And the mock scenario has a step "broken step" with status "failed" + And the mock scenario has a step "skipped step" with status "skipped" + When the TDD expected-fail handler processes the scenario + Then the scenario status should be "passed" + And the step "ok step" should have status "passed" + And the step "broken step" should have status "passed" + And the step "skipped step" should have status "passed" + And the step "broken step" should have error_message cleared + And the step "skipped step" should have error_message cleared + + # --- before_scenario hook_failed regression test (TC-2) --- + + Scenario: before_scenario sets hook_failed on invalid TDD tags + Given a mock scenario tagged "@tdd_expected_fail" with status "untested" + When before_scenario is called with the mock scenario + Then the scenario hook_failed flag should be True + And the scenario status should be "failed" + + # --- _install_tdd_expected_fail_patch (S2) --- + + Scenario: _install_tdd_expected_fail_patch sets the patched flag + When the TDD expected-fail patch installation status is checked + Then the Scenario class should have _tdd_run_patched set to True + + Scenario: _install_tdd_expected_fail_patch is idempotent + When _install_tdd_expected_fail_patch is called twice + Then the Scenario.run method should not be double-wrapped diff --git a/features/testing/__init__.py b/features/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/features/testing/tdd_expected_fail_demo.feature b/features/testing/tdd_expected_fail_demo.feature new file mode 100644 index 000000000..2cb29f104 --- /dev/null +++ b/features/testing/tdd_expected_fail_demo.feature @@ -0,0 +1,19 @@ +@mock_only +Feature: TDD expected-fail result inversion demo + Demonstrates the @tdd_expected_fail tag inverting a deliberately failing + scenario so it is reported as passed. This exercises the Scenario.run() + wrapper logic installed by _install_tdd_expected_fail_patch() in + features/environment.py. + See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification. + + Note: The "unexpected pass" path (where a @tdd_expected_fail scenario + passes and is forced to fail) cannot be tested as an integration scenario + because the forced failure would break the test suite. That path is + tested via mock-based unit tests in tdd_tag_validation.feature and via + real Scenario objects in tdd_expected_fail_infrastructure.feature. + + @tdd_bug @tdd_bug_999 @tdd_expected_fail + Scenario: Demo bug 999 expected failure is inverted to pass + Given tdd demo a step that always succeeds + When tdd demo a deliberately failing assertion is executed + Then tdd demo this step is never reached diff --git a/features/testing/tdd_tag_validation.feature b/features/testing/tdd_tag_validation.feature new file mode 100644 index 000000000..6ba3531d8 --- /dev/null +++ b/features/testing/tdd_tag_validation.feature @@ -0,0 +1,126 @@ +@mock_only +Feature: TDD bug-capture tag validation + As a developer writing TDD bug-capture tests + I want the environment hooks to validate my tag combinations + So that misconfigured TDD tests are caught before they run + + # Tests for the validate_tdd_tags() and should_invert_result() helper + # functions defined in features/environment.py. + # See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification. + + # --- validate_tdd_tags: valid combinations --- + + Scenario: Valid TDD tags with all three tags present + Given tdd tags a tag set with "tdd_bug, tdd_bug_123, tdd_expected_fail" + When tdd tags I validate the tag set + Then tdd tags validation should pass + + Scenario: Valid TDD tags with tdd_bug and tdd_bug_N only + Given tdd tags a tag set with "tdd_bug, tdd_bug_456" + When tdd tags I validate the tag set + Then tdd tags validation should pass + + Scenario: Valid TDD tags with tdd_bug only + Given tdd tags a tag set with "tdd_bug" + When tdd tags I validate the tag set + Then tdd tags validation should pass + + Scenario: Valid TDD tags with no TDD tags at all + Given tdd tags a tag set with "wip, slow" + When tdd tags I validate the tag set + Then tdd tags validation should pass + + Scenario: Valid TDD tags with empty tag set + Given tdd tags an empty tag set + When tdd tags I validate the tag set + Then tdd tags validation should pass + + # --- validate_tdd_tags: invalid combinations --- + + Scenario: Invalid TDD tags tdd_bug_N without tdd_bug raises error + Given tdd tags a tag set with "tdd_bug_123" + When tdd tags I validate the tag set + Then tdd tags validation should fail with error containing "@tdd_bug" + + Scenario: Invalid TDD tags tdd_expected_fail without tdd_bug raises error + Given tdd tags a tag set with "tdd_expected_fail, tdd_bug_123" + When tdd tags I validate the tag set + Then tdd tags validation should fail with error containing "@tdd_bug" + + Scenario: Invalid TDD tags tdd_expected_fail without tdd_bug_N raises error + Given tdd tags a tag set with "tdd_expected_fail, tdd_bug" + When tdd tags I validate the tag set + Then tdd tags validation should fail with error containing "@tdd_bug_" + + Scenario: Invalid TDD tags tdd_expected_fail alone raises error + Given tdd tags a tag set with "tdd_expected_fail" + When tdd tags I validate the tag set + Then tdd tags validation should fail with error containing "@tdd_bug" + + Scenario: Invalid TDD tags multiple tdd_bug_N without tdd_bug raises error + Given tdd tags a tag set with "tdd_bug_100, tdd_bug_200" + When tdd tags I validate the tag set + Then tdd tags validation should fail with error containing "@tdd_bug" + + # --- should_invert_result --- + + Scenario: should_invert_result returns True when tdd_expected_fail is present + Given tdd tags a tag set with "tdd_bug, tdd_bug_42, tdd_expected_fail" + Then tdd tags should_invert_result should return true + + Scenario: should_invert_result returns False when tdd_expected_fail is absent + Given tdd tags a tag set with "tdd_bug, tdd_bug_42" + Then tdd tags should_invert_result should return false + + Scenario: should_invert_result returns False for empty tag set + Given tdd tags an empty tag set + Then tdd tags should_invert_result should return false + + # --- apply_tdd_inversion: expected failure inverted to pass (happy path) --- + + Scenario: apply_tdd_inversion inverts expected AssertionError failure to pass + Given tdd inversion a mock scenario that fails with AssertionError and tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed True + Then tdd inversion the result should be False indicating expected failure was inverted + And tdd inversion the scenario status should be passed + And tdd inversion all failed and skipped steps should be reset to passed + And tdd inversion all step error_messages should be cleared + + # --- apply_tdd_inversion: unexpected pass forces failure --- + + Scenario: apply_tdd_inversion forces failure when tdd_expected_fail scenario passes + Given tdd inversion a mock scenario that passes with tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed False + Then tdd inversion the result should be True indicating forced failure + And tdd inversion the scenario status should be failed + And tdd inversion the last step should have a synthetic error message + And tdd inversion the last step should have a synthetic error_message text + + # --- apply_tdd_inversion: hook-error guard --- + + Scenario: apply_tdd_inversion does not invert when hook_failed is True + Given tdd inversion a mock scenario with hook_failed and tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed True + Then tdd inversion the result should be True indicating failure was not inverted + + # --- apply_tdd_inversion: dry-run guard --- + + Scenario: apply_tdd_inversion does not invert during dry-run mode + Given tdd inversion a mock scenario in dry-run mode with tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed False + Then tdd inversion the result should be False indicating no inversion occurred + + # --- apply_tdd_inversion: non-assertion exception guard --- + + Scenario: apply_tdd_inversion does not invert non-AssertionError exceptions + Given tdd inversion a mock scenario with a RuntimeError and tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed True + Then tdd inversion the result should be True indicating failure was not inverted + + # --- apply_tdd_inversion: empty steps edge case --- + + Scenario: apply_tdd_inversion handles unexpected pass with no steps + Given tdd inversion a mock scenario with no steps and tdd_expected_fail tags + When tdd inversion apply_tdd_inversion is called with failed False + Then tdd inversion the result should be True indicating forced failure + And tdd inversion the scenario status should be failed diff --git a/robot/helper_tdd_tag_validation.py b/robot/helper_tdd_tag_validation.py new file mode 100644 index 000000000..7b218c656 --- /dev/null +++ b/robot/helper_tdd_tag_validation.py @@ -0,0 +1,283 @@ +"""Helper script for tdd_tag_validation.robot integration tests. + +Each subcommand is a self-contained check that exercises the TDD +bug-capture tag validation and result-inversion logic defined in +``features/environment.py``. Sentinels are printed on success so +the Robot test case can assert correct behaviour. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import MagicMock + +# Ensure the project root is importable so ``features.environment`` resolves. +_ROOT = str(Path(__file__).resolve().parents[1]) +if _ROOT not in sys.path: + sys.path.insert(0, _ROOT) + +from behave.model import Status # noqa: E402 +from features.environment import ( # noqa: E402 + _UNEXPECTED_PASS_MSG, + apply_tdd_inversion, + should_invert_result, + validate_tdd_tags, +) + +# --------------------------------------------------------------------------- +# Mock helpers +# --------------------------------------------------------------------------- + + +def _make_mock_scenario( + tags: list[str], + steps_passed: bool = True, + hook_failed: bool = False, + was_dry_run: bool = False, + step_exception: BaseException | None = None, +) -> MagicMock: + """Build a lightweight mock ``Scenario`` for ``apply_tdd_inversion``.""" + scenario = MagicMock() + scenario.effective_tags = tags + scenario.name = "mock-scenario" + scenario.hook_failed = hook_failed + scenario.was_dry_run = was_dry_run + + mock_step = MagicMock() + if steps_passed: + mock_step.status = Status.passed + mock_step.exception = None + else: + mock_step.status = Status.failed + mock_step.exception = ( + step_exception + if step_exception is not None + else AssertionError("simulated assertion failure") + ) + scenario.all_steps = [mock_step] + return scenario + + +# --------------------------------------------------------------------------- +# Subcommands — validate_tdd_tags +# --------------------------------------------------------------------------- + + +def validate_tags_valid_combos() -> None: + """Verify valid tag combinations raise no error.""" + valid_sets: list[set[str]] = [ + {"tdd_bug", "tdd_bug_42"}, + {"tdd_bug", "tdd_bug_42", "tdd_expected_fail"}, + {"tdd_bug"}, + {"some_other_tag"}, + set(), + ] + for tag_set in valid_sets: + try: + validate_tdd_tags(tag_set) + except ValueError as exc: + print( + f"FAIL: valid tag set {tag_set} raised ValueError: {exc}", + file=sys.stderr, + ) + sys.exit(1) + print("validate-tags-valid-combos-ok") + + +def validate_tags_bug_n_without_bug() -> None: + """Verify @tdd_bug_ without @tdd_bug raises ValueError.""" + try: + validate_tdd_tags({"tdd_bug_42"}) + except ValueError: + print("validate-tags-bug-n-without-bug-ok") + return + print("FAIL: expected ValueError for tdd_bug_42 without tdd_bug", file=sys.stderr) + sys.exit(1) + + +def validate_tags_expected_fail_missing_bug() -> None: + """Verify @tdd_expected_fail without @tdd_bug raises ValueError.""" + try: + validate_tdd_tags({"tdd_expected_fail", "tdd_bug_42"}) + except ValueError: + print("validate-tags-expected-fail-missing-bug-ok") + return + print( + "FAIL: expected ValueError for tdd_expected_fail without tdd_bug", + file=sys.stderr, + ) + sys.exit(1) + + +def validate_tags_expected_fail_missing_bug_n() -> None: + """Verify @tdd_expected_fail without @tdd_bug_ raises ValueError.""" + try: + validate_tdd_tags({"tdd_expected_fail", "tdd_bug"}) + except ValueError: + print("validate-tags-expected-fail-missing-bug-n-ok") + return + print( + "FAIL: expected ValueError for tdd_expected_fail without tdd_bug_", + file=sys.stderr, + ) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Subcommands — should_invert_result +# --------------------------------------------------------------------------- + + +def should_invert_with_expected_fail() -> None: + """Verify should_invert_result returns True when @tdd_expected_fail present.""" + result = should_invert_result({"tdd_bug", "tdd_bug_42", "tdd_expected_fail"}) + if result is not True: + print(f"FAIL: expected True, got {result}", file=sys.stderr) + sys.exit(1) + print("should-invert-with-expected-fail-ok") + + +def should_not_invert_without_expected_fail() -> None: + """Verify should_invert_result returns False without @tdd_expected_fail.""" + result = should_invert_result({"tdd_bug", "tdd_bug_42"}) + if result is not False: + print(f"FAIL: expected False, got {result}", file=sys.stderr) + sys.exit(1) + print("should-not-invert-without-expected-fail-ok") + + +# --------------------------------------------------------------------------- +# Subcommands — apply_tdd_inversion +# --------------------------------------------------------------------------- + + +def inversion_expected_fail() -> None: + """Verify expected failure (failed=True) is inverted to passed.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"], + steps_passed=False, + ) + result = apply_tdd_inversion(scenario, failed=True) + if result is not False: + print(f"FAIL: expected False (inverted), got {result}", file=sys.stderr) + sys.exit(1) + scenario.set_status.assert_called_with(Status.passed) + print("inversion-expected-fail-ok") + + +def inversion_unexpected_pass() -> None: + """Verify unexpected pass (failed=False) is inverted to failure.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"], + steps_passed=True, + ) + result = apply_tdd_inversion(scenario, failed=False) + if result is not True: + print(f"FAIL: expected True (forced failure), got {result}", file=sys.stderr) + sys.exit(1) + scenario.set_status.assert_called_with(Status.failed) + # Verify synthetic error is attached to last step + last_step = scenario.all_steps[-1] + if last_step.status != Status.failed: + print( + f"FAIL: last step status should be failed, got {last_step.status}", + file=sys.stderr, + ) + sys.exit(1) + if not isinstance(last_step.exception, AssertionError): + print( + f"FAIL: expected AssertionError, got {type(last_step.exception)}", + file=sys.stderr, + ) + sys.exit(1) + if str(last_step.exception) != _UNEXPECTED_PASS_MSG: + print(f"FAIL: unexpected message: {last_step.exception}", file=sys.stderr) + sys.exit(1) + print("inversion-unexpected-pass-ok") + + +def inversion_hook_error_guard() -> None: + """Verify hook_failed prevents inversion.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"], + steps_passed=False, + hook_failed=True, + ) + result = apply_tdd_inversion(scenario, failed=True) + if result is not True: + print(f"FAIL: expected True (not inverted), got {result}", file=sys.stderr) + sys.exit(1) + print("inversion-hook-error-guard-ok") + + +def inversion_dry_run_guard() -> None: + """Verify dry-run mode prevents inversion.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"], + steps_passed=True, + was_dry_run=True, + ) + result = apply_tdd_inversion(scenario, failed=False) + if result is not False: + print(f"FAIL: expected False (not inverted), got {result}", file=sys.stderr) + sys.exit(1) + print("inversion-dry-run-guard-ok") + + +def inversion_non_assertion_guard() -> None: + """Verify non-AssertionError exceptions prevent inversion.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"], + steps_passed=False, + step_exception=RuntimeError("connection lost"), + ) + result = apply_tdd_inversion(scenario, failed=True) + if result is not True: + print(f"FAIL: expected True (not inverted), got {result}", file=sys.stderr) + sys.exit(1) + print("inversion-non-assertion-guard-ok") + + +def inversion_no_tag_passthrough() -> None: + """Verify scenarios without @tdd_expected_fail are not modified.""" + scenario = _make_mock_scenario( + tags=["tdd_bug", "tdd_bug_42"], + steps_passed=False, + ) + result = apply_tdd_inversion(scenario, failed=True) + if result is not True: + print(f"FAIL: expected True (passthrough), got {result}", file=sys.stderr) + sys.exit(1) + # set_status should NOT have been called + scenario.set_status.assert_not_called() + print("inversion-no-tag-passthrough-ok") + + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + +_COMMANDS = { + "validate_tags_valid_combos": validate_tags_valid_combos, + "validate_tags_bug_n_without_bug": validate_tags_bug_n_without_bug, + "validate_tags_expected_fail_missing_bug": validate_tags_expected_fail_missing_bug, + "validate_tags_expected_fail_missing_bug_n": ( + validate_tags_expected_fail_missing_bug_n + ), + "should_invert_with_expected_fail": should_invert_with_expected_fail, + "should_not_invert_without_expected_fail": should_not_invert_without_expected_fail, + "inversion_expected_fail": inversion_expected_fail, + "inversion_unexpected_pass": inversion_unexpected_pass, + "inversion_hook_error_guard": inversion_hook_error_guard, + "inversion_dry_run_guard": inversion_dry_run_guard, + "inversion_non_assertion_guard": inversion_non_assertion_guard, + "inversion_no_tag_passthrough": inversion_no_tag_passthrough, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + print(f"Commands: {list(_COMMANDS)}", file=sys.stderr) + sys.exit(1) + _COMMANDS[sys.argv[1]]() diff --git a/robot/tdd_tag_validation.robot b/robot/tdd_tag_validation.robot new file mode 100644 index 000000000..dc58729a7 --- /dev/null +++ b/robot/tdd_tag_validation.robot @@ -0,0 +1,134 @@ +*** Settings *** +Documentation Integration tests for TDD bug-capture tag validation and +... @tdd_expected_fail result inversion logic defined in +... features/environment.py. Covers validate_tdd_tags(), +... should_invert_result(), and apply_tdd_inversion() including +... all guard paths (hook errors, dry-run, non-assertion exceptions). +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_tdd_tag_validation.py + +*** Test Cases *** +Valid Tag Combinations Are Accepted + [Documentation] Verify valid TDD tag sets raise no errors + [Tags] testing tdd validation + ${result}= Run Process ${PYTHON} ${HELPER} validate_tags_valid_combos + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validate-tags-valid-combos-ok + +Bug N Without Bug Tag Raises Error + [Documentation] Verify @tdd_bug_ without @tdd_bug raises ValueError + [Tags] testing tdd validation + ${result}= Run Process ${PYTHON} ${HELPER} validate_tags_bug_n_without_bug + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validate-tags-bug-n-without-bug-ok + +Expected Fail Without Bug Tag Raises Error + [Documentation] Verify @tdd_expected_fail without @tdd_bug raises ValueError + [Tags] testing tdd validation + ${result}= Run Process ${PYTHON} ${HELPER} validate_tags_expected_fail_missing_bug + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validate-tags-expected-fail-missing-bug-ok + +Expected Fail Without Bug N Tag Raises Error + [Documentation] Verify @tdd_expected_fail without @tdd_bug_ raises ValueError + [Tags] testing tdd validation + ${result}= Run Process ${PYTHON} ${HELPER} validate_tags_expected_fail_missing_bug_n + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} validate-tags-expected-fail-missing-bug-n-ok + +Should Invert With Expected Fail Tag + [Documentation] Verify should_invert_result returns True with @tdd_expected_fail + [Tags] testing tdd inversion + ${result}= Run Process ${PYTHON} ${HELPER} should_invert_with_expected_fail + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} should-invert-with-expected-fail-ok + +Should Not Invert Without Expected Fail Tag + [Documentation] Verify should_invert_result returns False without @tdd_expected_fail + [Tags] testing tdd inversion + ${result}= Run Process ${PYTHON} ${HELPER} should_not_invert_without_expected_fail + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} should-not-invert-without-expected-fail-ok + +Inversion Expected Fail Is Inverted To Pass + [Documentation] Verify a failed @tdd_expected_fail scenario is inverted to passed + [Tags] testing tdd inversion + ${result}= Run Process ${PYTHON} ${HELPER} inversion_expected_fail + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-expected-fail-ok + +Inversion Unexpected Pass Forces Failure + [Documentation] Verify a passing @tdd_expected_fail scenario is forced to fail + ... with a synthetic error attached to the last step + [Tags] testing tdd inversion + ${result}= Run Process ${PYTHON} ${HELPER} inversion_unexpected_pass + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-unexpected-pass-ok + +Inversion Hook Error Guard Prevents Inversion + [Documentation] Verify hook_failed=True prevents result inversion + [Tags] testing tdd inversion guard + ${result}= Run Process ${PYTHON} ${HELPER} inversion_hook_error_guard + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-hook-error-guard-ok + +Inversion Dry Run Guard Prevents Inversion + [Documentation] Verify dry-run mode prevents result inversion + [Tags] testing tdd inversion guard + ${result}= Run Process ${PYTHON} ${HELPER} inversion_dry_run_guard + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-dry-run-guard-ok + +Inversion Non Assertion Guard Prevents Inversion + [Documentation] Verify non-AssertionError exceptions prevent result inversion + [Tags] testing tdd inversion guard + ${result}= Run Process ${PYTHON} ${HELPER} inversion_non_assertion_guard + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-non-assertion-guard-ok + +Inversion No Tag Passthrough + [Documentation] Verify scenarios without @tdd_expected_fail are not modified + [Tags] testing tdd inversion + ${result}= Run Process ${PYTHON} ${HELPER} inversion_no_tag_passthrough + ... cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} inversion-no-tag-passthrough-ok -- 2.52.0