feat(testing): implement @tdd_expected_fail tag handling in Behave environment #665
@@ -46,6 +46,14 @@
|
||||
protection, and `sync_results_to_host` for file-based result retrieval. Covered by
|
||||
Behave BDD scenarios, Robot Framework integration tests, ASV benchmarks, and
|
||||
`docs/reference/execution_environment.md`. (#515)
|
||||
- Implemented `@tdd_expected_fail` tag handling in Behave environment hooks. Added
|
||||
`validate_tdd_tags()` and `should_invert_result()` helper functions in
|
||||
`features/environment.py`. Scenarios tagged `@tdd_expected_fail` that fail have their
|
||||
result inverted to pass (expected failure); scenarios that unexpectedly pass are reported
|
||||
as failed with guidance to remove the tag. Tag validation enforces `@tdd_bug` +
|
||||
`@tdd_bug_<N>` prerequisites. Implemented via `Scenario.run()` monkey-patch in `before_all`.
|
||||
Includes 34 Behave BDD scenarios (19 tag-validation, 14 infrastructure, and
|
||||
1 demo) and 12 Robot Framework integration test cases. (#627)
|
||||
|
||||
|
||||
### Added
|
||||
|
||||
@@ -10,10 +10,7 @@ import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from behave.model import Scenario
|
||||
from behave.model_core import Status
|
||||
|
||||
_tdd_logger = logging.getLogger("behave.tdd_expected_fail")
|
||||
from behave.model import Scenario, Status
|
||||
|
||||
LANGSMITH_ENV_VARS = [
|
||||
"CLEVERAGENTS_LANGSMITH_ENABLED",
|
||||
@@ -33,6 +30,264 @@ LANGSMITH_ENV_VARS = [
|
||||
"LANGSMITH_USER_ID",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD Bug Test Tags — Three-Tag System
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD bug-capture tests use a three-tag system documented in
|
||||
# CONTRIBUTING.md > Bug Fix Workflow > TDD Bug Test Tags:
|
||||
#
|
||||
# @tdd_bug — Generic filter tag. Present on ALL TDD bug tests.
|
||||
# @tdd_bug_<N> — Issue reference (e.g. @tdd_bug_123). Links the
|
||||
# test to the specific Type/Bug issue it captures.
|
||||
# @tdd_expected_fail — Behavioral switch. When present, the test result
|
||||
# is inverted: a failure means the bug still exists
|
||||
# (reported as passed), and a pass means the bug was
|
||||
# fixed without removing the tag (reported as failed).
|
||||
#
|
||||
# The ``validate_tdd_tags`` and ``should_invert_result`` helpers below are
|
||||
# called from the ``before_scenario`` hook and the ``Scenario.run()``
|
||||
|
|
||||
# wrapper (installed in ``before_all``) respectively. They are extracted
|
||||
# as standalone functions so they can be unit-tested directly from Behave
|
||||
# step definitions. ``apply_tdd_inversion`` encapsulates the full
|
||||
# inversion logic and is likewise directly testable.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TDD_BUG_N_RE = re.compile(r"tdd_bug_\d+")
|
||||
|
||||
_tdd_logger = logging.getLogger("cleveragents.testing.tdd_tags")
|
||||
|
||||
|
||||
def validate_tdd_tags(tags: set[str]) -> None:
|
||||
"""Validate TDD bug-capture tag combinations.
|
||||
|
||||
Raises ``ValueError`` with a descriptive message when the tag set is
|
||||
inconsistent according to the rules in CONTRIBUTING.md:
|
||||
|
||||
* ``@tdd_bug_<N>`` requires ``@tdd_bug`` to also be present.
|
||||
* ``@tdd_expected_fail`` requires both ``@tdd_bug`` and at least one
|
||||
``@tdd_bug_<N>``.
|
||||
|
||||
Args:
|
||||
tags: The *effective* tags of a scenario (own tags + feature tags).
|
||||
|
||||
Raises:
|
||||
ValueError: If the tag combination violates the TDD tag rules.
|
||||
"""
|
||||
has_tdd_bug = "tdd_bug" in tags
|
||||
has_tdd_bug_n = any(_TDD_BUG_N_RE.fullmatch(t) for t in tags)
|
||||
has_expected_fail = "tdd_expected_fail" in tags
|
||||
|
||||
if has_tdd_bug_n and not has_tdd_bug:
|
||||
bug_n_tags = sorted(t for t in tags if _TDD_BUG_N_RE.fullmatch(t))
|
||||
raise ValueError(
|
||||
f"Scenario has {', '.join('@' + t for t in bug_n_tags)} but is "
|
||||
f"missing the required @tdd_bug tag. All TDD bug tests must "
|
||||
f"include @tdd_bug. See CONTRIBUTING.md > TDD Bug Test Tags."
|
||||
)
|
||||
|
||||
if has_expected_fail:
|
||||
missing: list[str] = []
|
||||
if not has_tdd_bug:
|
||||
missing.append("@tdd_bug")
|
||||
if not has_tdd_bug_n:
|
||||
missing.append("@tdd_bug_<N>")
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Scenario has @tdd_expected_fail but is missing required "
|
||||
f"tag(s): {', '.join(missing)}. @tdd_expected_fail requires "
|
||||
f"both @tdd_bug and at least one @tdd_bug_<N>. "
|
||||
f"See CONTRIBUTING.md > TDD Bug Test Tags."
|
||||
)
|
||||
|
||||
|
||||
def should_invert_result(tags: set[str]) -> bool:
|
||||
"""Return ``True`` if the scenario result should be inverted.
|
||||
|
||||
A scenario result is inverted when ``@tdd_expected_fail`` is present in
|
||||
the effective tags, indicating the test captures a known bug that has not
|
||||
yet been fixed.
|
||||
|
||||
Args:
|
||||
tags: The *effective* tags of a scenario (own tags + feature tags).
|
||||
"""
|
||||
return "tdd_expected_fail" in tags
|
||||
|
||||
|
||||
_UNEXPECTED_PASS_MSG = (
|
||||
"Bug appears to be fixed. Remove the @tdd_expected_fail tag "
|
||||
"from this scenario and verify the fix through the bug fix "
|
||||
"workflow. See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
|
||||
|
||||
def apply_tdd_inversion(scenario: Any, failed: bool) -> bool:
|
||||
"""Apply TDD expected-fail result inversion to a completed scenario.
|
||||
|
||||
Encapsulates the full inversion logic so it can be unit-tested directly
|
||||
from Behave step definitions (using mock scenario objects).
|
||||
|
||||
Guards against infrastructure errors and dry-run mode before inverting:
|
||||
|
||||
* **Hook/cleanup errors** (``scenario.hook_failed``): never inverted —
|
||||
infrastructure failures must propagate regardless of tags.
|
||||
* **Dry-run mode** (``scenario.was_dry_run``): never inverted — no test
|
||||
actually executed, so the result is meaningless.
|
||||
* **Non-assertion exceptions**: a step that failed with an exception
|
||||
other than ``AssertionError`` likely indicates an infrastructure
|
||||
problem, not the captured bug — a warning is logged and the result
|
||||
|
hurui200320
commented
Major (Still Open) — Dry-run false failures. Behave sets Add after the **Major (Still Open) — Dry-run false failures.** Behave sets `self.was_dry_run = True` when `runner.config.dry_run` is active (verified in Behave source). In dry-run mode, no steps execute, `_original_run` returns `False`, and this wrapper enters the 'unexpected pass' branch — incorrectly forcing failure on every `@tdd_expected_fail` scenario.
Add after the `_original_run` call:
```python
if getattr(self, 'was_dry_run', False):
return failed
```
|
||||
is *not* inverted.
|
||||
|
hurui200320
commented
Major — False failures in In dry-run mode, Behave skips hooks ( This is incorrect — no test actually ran, so the bug's status is unknown. Every Fix: Add a dry-run guard:
**Major — False failures in `--dry-run` mode**
In dry-run mode, Behave skips hooks (`if not runner.config.dry_run and run_scenario:`) and doesn't execute steps. `_original_run` returns `False` (no failures). The wrapper then enters the 'unexpected pass' branch at line 155 and forces a failure with "Bug appears to be fixed".
This is incorrect — no test actually ran, so the bug's status is unknown. Every `@tdd_expected_fail` scenario would fail during `--dry-run`, breaking test discovery.
**Fix:** Add a dry-run guard:
```python
if getattr(self, 'was_dry_run', False):
return failed
```
`was_dry_run` is set by Behave's `Scenario.run()` at the start of execution.
hurui200320
commented
Major (Issue 2) — Still unresolved: Dry-run false failures. Confirmed: Behave sets Needs: **Major (Issue 2) — Still unresolved:** Dry-run false failures.
Confirmed: Behave sets `self.was_dry_run = dry_run_scenario` at line ~10 of `Scenario.run()`. In dry-run mode, hooks are skipped and no steps execute, so `failed` is always `False`. The wrapper interprets this as 'unexpected pass' and forces failure.
Needs:
```python
if getattr(self, 'was_dry_run', False):
return failed
```
hurui200320
commented
Still unresolved — Dry-run false failure (Major) In Add before the inversion: **Still unresolved — Dry-run false failure (Major)**
In `--dry-run` mode, `_original_run` returns `False` (no failure) and `self.was_dry_run` is `True`. Without a guard, this falls through to the 'unexpected pass' branch at line 155, forcing every `@tdd_expected_fail` scenario to fail with a misleading message.
Add before the inversion:
```python
if getattr(self, 'was_dry_run', False):
return failed
```
|
||||
|
||||
When inverting expected failures, ``step.error_message`` is also cleared
|
||||
|
hurui200320
commented
Major — Hook errors masked by inversion If The wrapper needs to check **Major — Hook errors masked by inversion**
If `before_scenario` (lines 370-442), `after_scenario`, or context cleanup raises an exception on a `@tdd_expected_fail` scenario, the wrapper sees `failed=True` and flips it to passed. The hook error is silently swallowed.
The wrapper needs to check `self.hook_failed` or `self.status in (Status.hook_error, Status.cleanup_error)` and bail out without inverting:
```python
if self.hook_failed or self.status in (
Status.hook_error, Status.cleanup_error,
):
return failed
```
hurui200320
commented
Major (Still Open) — Hook errors masked. After before the **Major (Still Open) — Hook errors masked.** After `_original_run` returns, `self.hook_failed` is guaranteed to be set by Behave's `Scenario.run()` (verified via source inspection). Add:
```python
if self.hook_failed:
return failed
```
before the `should_invert_result` check. Without this, a `before_scenario` exception on a `@tdd_expected_fail` scenario is silently reported as passed.
hurui200320
commented
Still unresolved — Hook-error masking (Major) After the call to Behave source confirms before the inversion logic. **Still unresolved — Hook-error masking (Major)**
After the call to `_original_run`, `self.hook_failed` will be `True` if any hook raised an exception. The current code proceeds to invert the result, hiding the infrastructure failure.
Behave source confirms `self.hook_failed` is set at the top of `Scenario.run()` and toggled on hook failure. Add:
```python
if self.hook_failed:
return failed
```
before the inversion logic.
|
||||
alongside ``step.exception`` and ``step.exc_traceback`` to prevent stale
|
||||
|
hurui200320
commented
Major (Blocking) — Hook errors silently masked by result inversion After I verified in Behave's Fix: Add before line 141: **Major (Blocking) — Hook errors silently masked by result inversion**
After `_original_run` returns, `self.hook_failed` is `True` if `before_scenario` or `after_scenario` raised an exception. The wrapper only checks `if failed:` and inverts it to passed, completely hiding the infrastructure error.
I verified in Behave's `Scenario.run()` source that `self.hook_failed` is set at the top of the method and checked at line ~32. It is available and reliable after `_original_run` returns.
**Fix:** Add before line 141:
```python
if self.hook_failed or self.status in (Status.hook_error, Status.cleanup_error):
return failed # Infrastructure errors must never be inverted
```
hurui200320
commented
Major (Issue 1) — Still unresolved: Hook/cleanup errors masked. After reading Behave's Needs a guard: **Major (Issue 1) — Still unresolved:** Hook/cleanup errors masked.
After reading Behave's `Scenario.run()` source, confirmed: after the `after_scenario` hook runs, if `self.hook_failed` is `True`, Behave sets `self.status = Status.hook_error` and `failed = True`. Similarly, if context `_pop()` raises, `self.status = Status.cleanup_error` and `failed = True`. Both cases reach the wrapper with `failed=True`, which then inverts to passed.
Needs a guard:
```python
if self.hook_failed or self.status in (Status.hook_error, Status.cleanup_error):
return failed
```
|
||||
failure text from leaking to JUnit XML reporters and custom formatters.
|
||||
|
||||
Args:
|
||||
|
hurui200320
commented
Minor — Any exception type treated as 'bug still exists' The wrapper inverts any step failure, whether it's an Consider verifying that failed steps have **Minor — Any exception type treated as 'bug still exists'**
The wrapper inverts any step failure, whether it's an `AssertionError` (expected for TDD bug tests) or a `RuntimeError`, `TypeError`, `ConnectionError`, etc. A genuine infrastructure failure during step execution would be silently treated as "bug still exists" rather than flagged as an error.
Consider verifying that failed steps have `AssertionError` before inverting, and logging a warning (without inverting) for non-assertion exceptions.
hurui200320
commented
Minor (Still Open) — Any exception type inverted. This branch inverts all failures regardless of exception type. A **Minor (Still Open) — Any exception type inverted.** This branch inverts all failures regardless of exception type. A `ConnectionError` or `TypeError` during step execution would be silently treated as 'bug still exists'. Consider guarding:
```python
for step in self.all_steps:
if step.status == Status.failed:
if step.exception and not isinstance(step.exception, AssertionError):
_tdd_logger.warning(
"Non-assertion exception in expected-fail scenario '%s' "
"step '%s': %s — not inverting.",
self.name, step.name, step.exception,
)
return failed
```
|
||||
scenario: A Behave ``Scenario`` (or compatible mock) with at least
|
||||
``effective_tags``, ``all_steps``, ``name``, ``hook_failed``,
|
||||
``was_dry_run``, ``clear_status()``, and ``set_status()``
|
||||
|
hurui200320
commented
Minor — Diagnostic info lost without logging The exception and traceback are cleared without any record. If a developer needs to verify that the expected failure is the correct failure (the right Consider logging at DEBUG before clearing: **Minor — Diagnostic info lost without logging**
The exception and traceback are cleared without any record. If a developer needs to verify that the expected failure is the *correct* failure (the right `AssertionError`, not an unrelated crash), this info is gone.
Consider logging at DEBUG before clearing:
```python
_tdd_logger.debug(
"Clearing expected-fail exception for step '%s': %s",
step.name, step.exception,
)
```
hurui200320
commented
Minor — Diagnostic info lost without logging The exception and traceback are cleared here without any record. If a developer needs to verify the expected failure is the correct failure (not an unrelated crash), there's no trace. Fix: Log at DEBUG before clearing: **Minor — Diagnostic info lost without logging**
The exception and traceback are cleared here without any record. If a developer needs to verify the expected failure is the *correct* failure (not an unrelated crash), there's no trace.
**Fix:** Log at DEBUG before clearing:
```python
_tdd_logger.debug("Clearing expected-fail exception for step '%s': %s", step.name, step.exception)
```
hurui200320
commented
Minor (Issue 3) — Still unresolved: These lines discard **Minor (Issue 3) — Still unresolved:** These lines discard `step.exception` and `step.exc_traceback` without logging. Add `_tdd_logger.debug(...)` before clearing so developers can verify the failure was the expected one.
hurui200320
commented
Minor (Still Open) — Diagnostic info lost. Before setting **Minor (Still Open) — Diagnostic info lost.** Before setting `step.exception = None`, consider `_tdd_logger.debug("Inverting expected-fail step '%s': %s", step.name, step.exception)` so developers can verify the *correct* failure is being inverted during verbose/debug runs.
hurui200320
commented
Minor — Diagnostic info silently discarded Clearing the exception and traceback here loses the only record of which failure occurred. For expected-fail scenarios, knowing the exception helps developers verify they're testing the right bug. Consider: before the assignment. **Minor — Diagnostic info silently discarded**
Clearing the exception and traceback here loses the only record of *which* failure occurred. For expected-fail scenarios, knowing the exception helps developers verify they're testing the right bug.
Consider:
```python
_tdd_logger.debug(
"Clearing expected-fail exception for step '%s': %s",
step.name, step.exception,
)
```
before the assignment.
|
||||
attributes.
|
||||
failed: The boolean return value of the original ``Scenario.run()``.
|
||||
|
||||
Returns:
|
||||
The (possibly inverted) failure status to be returned to the runner.
|
||||
|
hurui200320
commented
Major (Blocking) — In dry-run mode, Behave sets I verified in Behave's Fix: Add after the **Major (Blocking) — `--dry-run` mode falsely fails all `@tdd_expected_fail` scenarios**
In dry-run mode, Behave sets `self.was_dry_run = True` and skips step execution. `_original_run` returns `False`. The wrapper enters the 'unexpected pass' branch here and forces failure with "Bug appears to be fixed" — even though no test actually ran.
I verified in Behave's `Scenario.run()` source that `self.was_dry_run = dry_run_scenario` is set at line ~10.
**Fix:** Add after the `should_invert_result` check:
```python
if getattr(self, 'was_dry_run', False):
return failed
```
|
||||
"""
|
||||
if not should_invert_result(set(scenario.effective_tags)):
|
||||
return failed
|
||||
|
||||
# Guard: never invert infrastructure / hook errors.
|
||||
if getattr(scenario, "hook_failed", False):
|
||||
return failed
|
||||
|
||||
# Guard: no-op during dry-run — no test actually executed.
|
||||
|
hurui200320
commented
Minor (Issue 5, new) — No visible failure reason in test output. This Consider setting a synthetic error on the last step so the reason appears in standard output rather than only in logs. **Minor (Issue 5, new) — No visible failure reason in test output.**
This `set_status(Status.failed)` + `return True` forces a failure, but no step is marked as failed and no error text is attached. Behave formatters display the failed step's exception — without one, the output shows 'FAILED' with no explanation.
Consider setting a synthetic error on the last step so the reason appears in standard output rather than only in logs.
|
||||
if getattr(scenario, "was_dry_run", False):
|
||||
return failed
|
||||
|
||||
# Materialise all_steps once — some Behave versions return an
|
||||
# iterator (list_iterator / itertools.chain) instead of a list.
|
||||
all_steps = list(scenario.all_steps)
|
||||
|
||||
if failed:
|
||||
# Guard: do not invert non-assertion exceptions — they likely
|
||||
# indicate an infrastructure problem, not the captured bug.
|
||||
for step in all_steps:
|
||||
if (
|
||||
step.status == Status.failed
|
||||
and step.exception is not None
|
||||
and not isinstance(step.exception, AssertionError)
|
||||
):
|
||||
_tdd_logger.warning(
|
||||
"Non-assertion exception in expected-fail scenario "
|
||||
"'%s' step '%s': %s — not inverting.",
|
||||
scenario.name,
|
||||
step.name,
|
||||
step.exception,
|
||||
)
|
||||
return failed
|
||||
|
||||
# Expected failure — the bug still exists. Reset the failed
|
||||
# and skipped steps so the scenario is reported as passed.
|
||||
# (When a step fails, subsequent steps are marked as skipped;
|
||||
# both must be set to ``Status.passed`` for accurate summary
|
||||
# counts and consistent ``compute_status()`` behaviour.)
|
||||
for step in all_steps:
|
||||
if step.status in (Status.failed, Status.skipped):
|
||||
_tdd_logger.debug(
|
||||
"Clearing expected-fail exception for step '%s': %s",
|
||||
step.name,
|
||||
step.exception,
|
||||
)
|
||||
step.status = Status.passed
|
||||
step.exception = None
|
||||
step.exc_traceback = None
|
||||
step.error_message = None
|
||||
scenario.clear_status()
|
||||
scenario.set_status(Status.passed)
|
||||
return False # Not a failure for the runner
|
||||
|
||||
# Unexpected pass — the bug appears to be fixed but the
|
||||
# @tdd_expected_fail tag has not been removed. Force a failure
|
||||
# so CI blocks the PR until the tag is cleaned up.
|
||||
_tdd_logger.warning(
|
||||
"Bug appears to be fixed. Remove the @tdd_expected_fail "
|
||||
"tag from scenario '%s' and verify the fix through the "
|
||||
"bug fix workflow. See CONTRIBUTING.md > Bug Fix Workflow.",
|
||||
scenario.name,
|
||||
)
|
||||
# Attach a synthetic error to the last step so the failure reason
|
||||
# appears in standard Behave output (formatters show the failed
|
||||
# step's exception text).
|
||||
|
brent.edwards
commented
P1:must-fix (F1) — This function is no longer called from Fix: Either update the old infrastructure tests to exercise **P1:must-fix (F1)** — This function is no longer called from `after_scenario` (the PR correctly moved inversion to the `Scenario.run()` wrapper). However, the 6 old scenarios in `tdd_expected_fail_infrastructure.feature` still call it directly. This creates a false-confidence coverage signal — tests pass but validate dead code with different semantics from the production `apply_tdd_inversion` (missing non-assertion guard, unconditional step clearing, no exception logging).
**Fix**: Either update the old infrastructure tests to exercise `apply_tdd_inversion` instead, or remove `handle_tdd_expected_fail` and the old tests entirely.
|
||||
if all_steps:
|
||||
last_step = all_steps[-1]
|
||||
last_step.status = Status.failed
|
||||
last_step.exception = AssertionError(_UNEXPECTED_PASS_MSG)
|
||||
last_step.error_message = "Assertion Failed: " + _UNEXPECTED_PASS_MSG
|
||||
scenario.set_status(Status.failed)
|
||||
return True # Force failure for the runner
|
||||
|
||||
|
||||
def handle_tdd_expected_fail(scenario: Any) -> None:
|
||||
"""Process a scenario through TDD expected-fail tag validation and inversion.
|
||||
|
||||
Public, testable entry point that encapsulates the full TDD expected-fail
|
||||
logic operating directly on a ``Scenario`` object:
|
||||
|
||||
1. **Tag validation** — rejects invalid TDD tag combinations by forcing
|
||||
the scenario to ``Status.failed``.
|
||||
2. **Result inversion** — delegates to :func:`apply_tdd_inversion` for
|
||||
valid ``@tdd_expected_fail`` scenarios, which inverts the result:
|
||||
a failure becomes passed (expected), a pass becomes failed
|
||||
(unexpected — the bug appears fixed).
|
||||
|
||||
|
brent.edwards
commented
P2:should-fix — Unlike **P2:should-fix** — Unlike `apply_tdd_inversion` (which guards against non-`AssertionError` exceptions at lines 171-182), `handle_tdd_expected_fail` blindly inverts all failures here. A `RuntimeError` from an infrastructure problem would be silently converted to a pass. Add the same non-assertion guard for parity.
|
||||
All guard logic (hook errors, dry-run, non-assertion exceptions) and
|
||||
step-level status manipulation are handled by :func:`apply_tdd_inversion`.
|
||||
|
||||
Args:
|
||||
scenario: A Behave ``Scenario`` (or compatible mock) with at least
|
||||
``effective_tags`` (or ``tags``), ``all_steps``, ``status``,
|
||||
``hook_failed``, ``was_dry_run``, ``clear_status()``, and
|
||||
``set_status()``.
|
||||
"""
|
||||
tags = set(getattr(scenario, "effective_tags", getattr(scenario, "tags", [])))
|
||||
|
||||
# Validate tags — force failure on invalid combinations.
|
||||
try:
|
||||
validate_tdd_tags(tags)
|
||||
except ValueError as exc:
|
||||
_tdd_logger.warning(
|
||||
"Invalid TDD tag combination on scenario '%s': %s",
|
||||
scenario.name,
|
||||
exc,
|
||||
)
|
||||
scenario.set_status(Status.failed)
|
||||
return
|
||||
|
||||
# Delegate inversion logic to the shared implementation.
|
||||
apply_tdd_inversion(scenario, scenario.status == Status.failed)
|
||||
|
||||
|
||||
def _install_tdd_expected_fail_patch() -> None:
|
||||
"""Monkey-patch ``Scenario.run`` to invert results for ``@tdd_expected_fail``.
|
||||
|
||||
Behave's ``Scenario.run()`` returns a local ``failed`` boolean that
|
||||
``after_scenario`` hooks cannot modify. To correctly invert the result
|
||||
(so that an expected-fail scenario is reported as passed to the runner),
|
||||
we wrap ``Scenario.run()`` with a thin post-processing layer that
|
||||
delegates to :func:`apply_tdd_inversion`.
|
||||
|
||||
The patch is installed once in ``before_all`` and is idempotent.
|
||||
"""
|
||||
if getattr(Scenario, "_tdd_run_patched", False):
|
||||
return # Already patched (e.g. forked worker reloading hooks)
|
||||
|
||||
_original_run = Scenario.run
|
||||
|
||||
def _tdd_aware_run(self: Any, runner: Any) -> bool:
|
||||
failed: bool = _original_run(self, runner)
|
||||
return apply_tdd_inversion(self, failed)
|
||||
|
||||
Scenario.run = _tdd_aware_run
|
||||
Scenario._tdd_run_patched = True
|
||||
|
||||
|
||||
def before_all(context):
|
||||
|
brent.edwards
commented
P1:must-fix (F2) — Fix: Restore **P1:must-fix (F2)** — `from behave.model import Scenario` moved inside function. Master had this at the top of the file (line 13). CONTRIBUTING.md §1289-1294 requires all imports at module level (only exception: `TYPE_CHECKING`). This is a regression.
**Fix**: Restore `from behave.model import Scenario` at the top alongside `from behave.model import Status`, and remove this inner import.
|
||||
"""Set up test environment before all tests."""
|
||||
@@ -85,6 +340,11 @@ def before_all(context):
|
||||
_ensure_template_db()
|
||||
_install_template_db_patch()
|
||||
|
||||
# --- TDD Expected-Fail Patch ---
|
||||
# Wrap Scenario.run() so @tdd_expected_fail inverts the result for
|
||||
# both the scenario status AND the runner's pass/fail return value.
|
||||
_install_tdd_expected_fail_patch()
|
||||
|
||||
|
||||
def _install_fast_sleep_patch() -> None:
|
||||
"""Cap ``time.sleep`` and ``asyncio.sleep`` at 10 ms for fast test execution.
|
||||
@@ -230,6 +490,18 @@ def _install_template_db_patch() -> None:
|
||||
|
||||
def before_scenario(context, scenario):
|
||||
"""Set up before each scenario."""
|
||||
# --- TDD Bug Tag Validation ---
|
||||
# Validate the three-tag system BEFORE any other setup so that
|
||||
# misconfigured TDD tests are caught immediately.
|
||||
# See CONTRIBUTING.md > TDD Bug Test Tags for the full specification.
|
||||
try:
|
||||
validate_tdd_tags(set(scenario.effective_tags))
|
||||
except ValueError as exc:
|
||||
scenario.hook_failed = True
|
||||
scenario.set_status(Status.failed)
|
||||
_tdd_logger.error("TDD TAG ERROR in %r: %s", scenario.name, exc)
|
||||
return
|
||||
|
||||
# Store original working directory
|
||||
context.original_cwd = os.getcwd()
|
||||
|
||||
|
brent.edwards
commented
P2:should-fix (F3) — Bare Suggested fix: Also: no test exercises this hook-error path. **P2:should-fix (F3)** — Bare `ValueError` propagating into Behave's hook dispatcher produces `HOOK-ERROR in before_scenario: ValueError: ...` which looks like an infrastructure crash, not a tag error. Master's explicit stderr messages were clearer.
**Suggested fix**:
```python
try:
validate_tdd_tags(set(scenario.effective_tags))
except ValueError as exc:
scenario.set_status(Status.failed)
_tdd_logger.error("TDD TAG ERROR in %r: %s", scenario.name, exc)
return
```
Also: no test exercises this hook-error path.
|
||||
@@ -317,96 +589,8 @@ def before_scenario(context, scenario):
|
||||
pass # Container not needed for all tests
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD expected-fail tag handler (see CONTRIBUTING.md §TDD Bug Test Tags)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TDD_BUG_N_RE = re.compile(r"^tdd_bug_\d+$")
|
||||
|
||||
|
||||
def handle_tdd_expected_fail(scenario: Scenario) -> None:
|
||||
"""Validate TDD tags and invert pass/fail for ``@tdd_expected_fail``.
|
||||
|
||||
Tag validation (per CONTRIBUTING.md — unconditional):
|
||||
|
||||
* Any scenario with ``@tdd_bug_<N>`` **must** also carry ``@tdd_bug``.
|
||||
Missing ``@tdd_bug`` causes the scenario to fail unconditionally.
|
||||
|
||||
Status inversion (only when ``@tdd_expected_fail`` is present):
|
||||
|
||||
* ``@tdd_expected_fail`` additionally requires ``@tdd_bug`` **and** at
|
||||
least one ``@tdd_bug_<N>`` tag.
|
||||
* **failed → passed** — the bug still triggers, which is expected.
|
||||
* **passed → failed** — the bug was fixed but the tag was not removed;
|
||||
this is an error that must be caught.
|
||||
"""
|
||||
tags = set(scenario.effective_tags)
|
||||
|
||||
# --- unconditional tag validation (F5) --------------------------------
|
||||
has_tdd_bug_n = any(_TDD_BUG_N_RE.match(t) for t in tags)
|
||||
if has_tdd_bug_n and "tdd_bug" not in tags:
|
||||
scenario.set_status(Status.failed)
|
||||
sys.stderr.write(
|
||||
f"TDD TAG ERROR: {scenario.name!r} — @tdd_bug_<N> requires @tdd_bug tag\n"
|
||||
)
|
||||
return
|
||||
|
||||
if "tdd_expected_fail" not in tags:
|
||||
return
|
||||
|
||||
# --- @tdd_expected_fail tag validation --------------------------------
|
||||
if "tdd_bug" not in tags:
|
||||
scenario.set_status(Status.failed)
|
||||
sys.stderr.write(
|
||||
f"TDD TAG ERROR: {scenario.name!r} — "
|
||||
"@tdd_expected_fail requires @tdd_bug tag\n"
|
||||
)
|
||||
return
|
||||
|
||||
if not has_tdd_bug_n:
|
||||
scenario.set_status(Status.failed)
|
||||
sys.stderr.write(
|
||||
f"TDD TAG ERROR: {scenario.name!r} — "
|
||||
"@tdd_expected_fail requires at least one @tdd_bug_<N> tag\n"
|
||||
)
|
||||
return
|
||||
|
||||
# --- status inversion -------------------------------------------------
|
||||
if scenario.status == Status.failed:
|
||||
# Log original failure details before inverting so CI logs show what
|
||||
# actually failed (N3 review finding).
|
||||
for step in scenario.steps:
|
||||
if step.status == Status.failed:
|
||||
_tdd_logger.info(
|
||||
"TDD inversion: %s — step %r failed: %s",
|
||||
scenario.name,
|
||||
step.name,
|
||||
step.error_message or "(no message)",
|
||||
)
|
||||
|
||||
# Bug still present — expected. Mark scenario and its failed/skipped
|
||||
# steps as passed so that summary counts are accurate.
|
||||
scenario.clear_status()
|
||||
scenario.set_status(Status.passed)
|
||||
for step in scenario.steps:
|
||||
if step.status in (Status.failed, Status.skipped):
|
||||
step.status = Status.passed
|
||||
elif scenario.status == Status.passed:
|
||||
# Bug was fixed but @tdd_expected_fail was not removed — error.
|
||||
scenario.set_status(Status.failed)
|
||||
sys.stderr.write(
|
||||
f"TDD TAG ERROR: {scenario.name!r} — "
|
||||
"scenario passed but still carries @tdd_expected_fail; "
|
||||
"remove the tag now that the bug is fixed\n"
|
||||
)
|
||||
|
||||
|
||||
def after_scenario(context, scenario):
|
||||
"""Clean up after each scenario."""
|
||||
# Handle TDD expected-fail inversion BEFORE cleanup (status is already set
|
||||
# by step execution; cleanup does not change it).
|
||||
handle_tdd_expected_fail(scenario)
|
||||
|
||||
# Return to original directory first
|
||||
if hasattr(context, "original_cwd"):
|
||||
os.chdir(context.original_cwd)
|
||||
@@ -511,8 +695,6 @@ def after_scenario(context, scenario):
|
||||
|
||||
# T6: Remove log handlers attached to the async-cleanup logger by
|
||||
# security_async_steps.py so handlers don't accumulate across scenarios.
|
||||
import logging
|
||||
|
||||
if hasattr(context, "log_handler"):
|
||||
async_logger = logging.getLogger("cleveragents.core.async_cleanup")
|
||||
async_logger.removeHandler(context.log_handler)
|
||||
@@ -521,3 +703,8 @@ def after_scenario(context, scenario):
|
||||
if hasattr(context, "bridge_loop"):
|
||||
with contextlib.suppress(Exception):
|
||||
context.bridge_loop.close()
|
||||
|
||||
# NOTE: TDD @tdd_expected_fail result inversion is handled by the
|
||||
# Scenario.run() wrapper installed in _install_tdd_expected_fail_patch(),
|
||||
# NOT in this hook. See before_all() and CONTRIBUTING.md > TDD Bug
|
||||
# Test Tags for the full specification.
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Step definitions used exclusively by ``tdd_expected_fail_demo.feature``.
|
||||
|
||||
Per CONTRIBUTING.md: steps used only by a particular feature file must live
|
||||
in a correspondingly named step definition file.
|
||||
|
||||
These steps provide deliberately-passing and deliberately-failing behaviour
|
||||
to exercise the ``Scenario.run()`` wrapper installed by
|
||||
``_install_tdd_expected_fail_patch()`` in ``features/environment.py``.
|
||||
|
||||
See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
|
||||
@given("tdd demo a step that always succeeds")
|
||||
def step_tdd_demo_success(context: Context) -> None:
|
||||
"""A trivial passing step."""
|
||||
context.tdd_demo_ran = True
|
||||
|
||||
|
||||
@when("tdd demo a deliberately failing assertion is executed")
|
||||
def step_tdd_demo_deliberate_fail(context: Context) -> None:
|
||||
"""This step deliberately fails to simulate a bug still being present.
|
||||
|
||||
The ``@tdd_expected_fail`` tag on the scenario causes the
|
||||
``Scenario.run()`` wrapper to invert this failure into a pass.
|
||||
"""
|
||||
msg = "Deliberate failure: bug #999 is still present (expected by TDD tag)"
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
@then("tdd demo this step is never reached")
|
||||
def step_tdd_demo_never_reached(context: Context) -> None:
|
||||
"""Placeholder step — unreachable because the previous step fails."""
|
||||
context.tdd_demo_unreachable = True
|
||||
@@ -1,18 +1,25 @@
|
||||
"""Step definitions for the TDD expected-fail handler infrastructure tests.
|
||||
|
||||
These scenarios exercise ``_handle_tdd_expected_fail`` directly using real
|
||||
Behave ``Scenario`` and ``Step`` objects (not the running test's own scenario)
|
||||
to verify both scenario-level and **step-level** status inversion.
|
||||
These scenarios exercise both ``apply_tdd_inversion`` (the production code
|
||||
path called by the ``Scenario.run()`` monkey-patch) and
|
||||
``handle_tdd_expected_fail`` (the standalone entry point) using real Behave
|
||||
``Scenario`` and ``Step`` objects to verify scenario-level and **step-level**
|
||||
status inversion, including guard paths (hook errors, dry-run, non-assertion
|
||||
exceptions).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.model import Scenario, Step
|
||||
from behave.model_core import Status
|
||||
from behave.model import Scenario, Status, Step
|
||||
from behave.runner import Context
|
||||
|
||||
from features.environment import handle_tdd_expected_fail
|
||||
from features.environment import (
|
||||
_install_tdd_expected_fail_patch,
|
||||
apply_tdd_inversion,
|
||||
before_scenario,
|
||||
handle_tdd_expected_fail,
|
||||
)
|
||||
|
||||
_STATUS_MAP: dict[str, Status] = {
|
||||
"failed": Status.failed,
|
||||
@@ -36,10 +43,27 @@ def step_mock_scenario(context: Context, tags: str, status: str) -> None:
|
||||
# Force the desired starting status.
|
||||
scenario.clear_status()
|
||||
scenario.set_status(_STATUS_MAP[status])
|
||||
# Initialise guard attributes (Behave sets these during Scenario.run()).
|
||||
scenario.hook_failed = False
|
||||
scenario.was_dry_run = False
|
||||
context.mock_scenario = scenario
|
||||
context.mock_steps = {}
|
||||
|
||||
|
||||
@given('a mock scenario tagged "{tags}" with status "{status}" and hook_failed')
|
||||
def step_mock_scenario_hook_failed(context: Context, tags: str, status: str) -> None:
|
||||
"""Build a real ``Scenario`` with ``hook_failed = True``."""
|
||||
step_mock_scenario(context, tags, status)
|
||||
context.mock_scenario.hook_failed = True
|
||||
|
||||
|
||||
@given('a mock scenario tagged "{tags}" with status "{status}" and was_dry_run')
|
||||
def step_mock_scenario_dry_run(context: Context, tags: str, status: str) -> None:
|
||||
"""Build a real ``Scenario`` with ``was_dry_run = True``."""
|
||||
step_mock_scenario(context, tags, status)
|
||||
context.mock_scenario.was_dry_run = True
|
||||
|
||||
|
||||
@given('the mock scenario has a step "{step_name}" with status "{status}"')
|
||||
def step_add_mock_step(context: Context, step_name: str, status: str) -> None:
|
||||
"""Add a real ``Step`` to the mock scenario with the requested status."""
|
||||
@@ -51,16 +75,90 @@ def step_add_mock_step(context: Context, step_name: str, status: str) -> None:
|
||||
name=step_name,
|
||||
)
|
||||
step.status = _STATUS_MAP[status]
|
||||
# Provide an error_message for failed steps so logging (N3) can use it.
|
||||
# Provide an error_message for failed steps so logging can use it.
|
||||
if status == "failed":
|
||||
step.error_message = f"simulated failure in {step_name!r}"
|
||||
step.exception = AssertionError(f"simulated failure in {step_name!r}")
|
||||
context.mock_scenario.steps.append(step)
|
||||
context.mock_steps[step_name] = step
|
||||
|
||||
|
||||
@given(
|
||||
'the mock scenario has an infrastructure-error step "{step_name}"'
|
||||
' with exception "{exc_type}"'
|
||||
)
|
||||
def step_add_mock_step_with_exception(
|
||||
context: Context, step_name: str, exc_type: str
|
||||
) -> None:
|
||||
"""Add a real ``Step`` with a specific non-assertion exception type."""
|
||||
step = Step(
|
||||
filename="<infrastructure-test>",
|
||||
line=1,
|
||||
keyword="Given",
|
||||
step_type="given",
|
||||
name=step_name,
|
||||
)
|
||||
step.status = Status.failed
|
||||
exc_classes: dict[str, type[BaseException]] = {
|
||||
"RuntimeError": RuntimeError,
|
||||
"TypeError": TypeError,
|
||||
"ConnectionError": ConnectionError,
|
||||
"AssertionError": AssertionError,
|
||||
}
|
||||
exc_cls = exc_classes.get(exc_type, RuntimeError)
|
||||
step.exception = exc_cls(f"simulated {exc_type} in {step_name!r}")
|
||||
step.error_message = f"simulated {exc_type} in {step_name!r}"
|
||||
context.mock_scenario.steps.append(step)
|
||||
context.mock_steps[step_name] = step
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("apply_tdd_inversion processes the scenario with failed True")
|
||||
def step_run_apply_inversion_failed(context: Context) -> None:
|
||||
"""Invoke ``apply_tdd_inversion`` with ``failed=True``."""
|
||||
context.apply_inversion_result = apply_tdd_inversion(
|
||||
context.mock_scenario, failed=True
|
||||
)
|
||||
|
||||
|
||||
@when("apply_tdd_inversion processes the scenario with failed False")
|
||||
def step_run_apply_inversion_not_failed(context: Context) -> None:
|
||||
"""Invoke ``apply_tdd_inversion`` with ``failed=False``."""
|
||||
context.apply_inversion_result = apply_tdd_inversion(
|
||||
context.mock_scenario, failed=False
|
||||
)
|
||||
|
||||
|
||||
@then("the apply_tdd_inversion result should be False")
|
||||
def step_check_inversion_result_false(context: Context) -> None:
|
||||
"""Assert ``apply_tdd_inversion`` returned ``False``."""
|
||||
assert context.apply_inversion_result is False, (
|
||||
f"Expected apply_tdd_inversion to return False, "
|
||||
f"got {context.apply_inversion_result!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the apply_tdd_inversion result should be True")
|
||||
def step_check_inversion_result_true(context: Context) -> None:
|
||||
"""Assert ``apply_tdd_inversion`` returned ``True``."""
|
||||
assert context.apply_inversion_result is True, (
|
||||
f"Expected apply_tdd_inversion to return True, "
|
||||
f"got {context.apply_inversion_result!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# handle_tdd_expected_fail steps (standalone entry point)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("the TDD expected-fail handler processes the scenario")
|
||||
def step_run_handler(context: Context) -> None:
|
||||
"""Invoke ``_handle_tdd_expected_fail`` on the mock scenario."""
|
||||
"""Invoke ``handle_tdd_expected_fail`` on the mock scenario."""
|
||||
handle_tdd_expected_fail(context.mock_scenario)
|
||||
|
||||
|
||||
@@ -81,3 +179,109 @@ def step_check_step_status(context: Context, step_name: str, expected: str) -> N
|
||||
assert actual == _STATUS_MAP[expected], (
|
||||
f"Expected step {step_name!r} status {expected!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('the step "{step_name}" should have error_message cleared')
|
||||
def step_check_error_message_cleared(context: Context, step_name: str) -> None:
|
||||
"""Assert a named step's ``error_message`` is ``None`` after inversion."""
|
||||
step = context.mock_steps[step_name]
|
||||
assert step.error_message is None, (
|
||||
f"Expected step {step_name!r} error_message to be None after "
|
||||
f"inversion, got {step.error_message!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('the step "{step_name}" should have a synthetic error_message')
|
||||
def step_check_synthetic_error_message(context: Context, step_name: str) -> None:
|
||||
"""Assert a named step's ``error_message`` contains the unexpected-pass text."""
|
||||
step = context.mock_steps[step_name]
|
||||
assert step.error_message is not None, (
|
||||
f"Expected step {step_name!r} to have an error_message set, got None"
|
||||
)
|
||||
assert "Bug appears to be fixed" in step.error_message, (
|
||||
f"Expected step {step_name!r} error_message to contain "
|
||||
f"'Bug appears to be fixed', got {step.error_message!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _install_tdd_expected_fail_patch tests (S2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("the TDD expected-fail patch installation status is checked")
|
||||
def step_check_patch_status(context: Context) -> None:
|
||||
"""Verify the patch was installed during ``before_all``."""
|
||||
# _install_tdd_expected_fail_patch is called in before_all, which
|
||||
# has already run by the time any scenario executes.
|
||||
context.patch_flag = getattr(Scenario, "_tdd_run_patched", False)
|
||||
|
||||
|
||||
@then("the Scenario class should have _tdd_run_patched set to True")
|
||||
def step_verify_patch_flag(context: Context) -> None:
|
||||
"""Assert the patch flag is set on the Scenario class."""
|
||||
assert context.patch_flag is True, (
|
||||
"Expected Scenario._tdd_run_patched to be True after "
|
||||
"_install_tdd_expected_fail_patch(), got False"
|
||||
)
|
||||
|
||||
|
||||
@when("_install_tdd_expected_fail_patch is called twice")
|
||||
def step_call_patch_twice(context: Context) -> None:
|
||||
"""Call ``_install_tdd_expected_fail_patch`` twice and capture ``run``.
|
||||
|
||||
Saves the current ``Scenario.run`` before the second call so the
|
||||
``Then`` step can verify identity. A cleanup handler restores the
|
||||
original method in case the idempotency guard is ever broken — this
|
||||
prevents a double-wrapped ``Scenario.run`` from leaking into
|
||||
subsequent scenarios in the same process.
|
||||
"""
|
||||
run_before = Scenario.run
|
||||
_install_tdd_expected_fail_patch()
|
||||
context.run_after_second_call = Scenario.run
|
||||
context.run_before_second_call = run_before
|
||||
|
||||
# TF-1 safety net: restore Scenario.run if the guard failed.
|
||||
def _restore_run() -> None:
|
||||
Scenario.run = run_before
|
||||
|
||||
context._cleanup_handlers.append(_restore_run)
|
||||
|
||||
|
||||
@then("the Scenario.run method should not be double-wrapped")
|
||||
def step_verify_no_double_wrap(context: Context) -> None:
|
||||
"""Assert a second call to the patch does not double-wrap ``Scenario.run``."""
|
||||
assert context.run_after_second_call is context.run_before_second_call, (
|
||||
"Expected Scenario.run to remain the same after a second call to "
|
||||
"_install_tdd_expected_fail_patch() (idempotency), but it was "
|
||||
"replaced — indicating double-wrapping"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# before_scenario hook_failed regression test (TC-2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("before_scenario is called with the mock scenario")
|
||||
def step_call_before_scenario(context: Context) -> None:
|
||||
"""Invoke ``before_scenario`` with the mock scenario.
|
||||
|
||||
Uses a lightweight mock ``context`` so the hook's non-TDD setup
|
||||
(database paths, container overrides, etc.) runs harmlessly.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
mock_context = MagicMock()
|
||||
mock_context._cleanup_handlers = []
|
||||
mock_context._scenario_db_paths = []
|
||||
before_scenario(mock_context, context.mock_scenario)
|
||||
|
||||
|
||||
@then("the scenario hook_failed flag should be True")
|
||||
def step_check_hook_failed_true(context: Context) -> None:
|
||||
"""Assert the scenario's ``hook_failed`` attribute is ``True``."""
|
||||
assert context.mock_scenario.hook_failed is True, (
|
||||
"Expected scenario.hook_failed to be True after before_scenario "
|
||||
"processes invalid TDD tags, got False"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Step definitions for TDD bug-capture tag validation scenarios.
|
||||
|
||||
Tests the ``validate_tdd_tags()``, ``should_invert_result()``, and
|
||||
``apply_tdd_inversion()`` helper functions defined in
|
||||
``features/environment.py``.
|
||||
|
||||
All step names are prefixed with ``tdd tags`` or ``tdd inversion`` to avoid
|
||||
``AmbiguousStep`` conflicts with existing steps.
|
||||
|
||||
See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.model import Status
|
||||
from behave.runner import Context
|
||||
|
||||
from features.environment import (
|
||||
apply_tdd_inversion,
|
||||
should_invert_result,
|
||||
validate_tdd_tags,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag-set construction helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('tdd tags a tag set with "{tags_csv}"')
|
||||
def step_given_tag_set(context: Context, tags_csv: str) -> None:
|
||||
"""Parse a comma-separated tag list into a set on the context."""
|
||||
context.tdd_tag_set = {t.strip() for t in tags_csv.split(",") if t.strip()}
|
||||
|
||||
|
||||
@given("tdd tags an empty tag set")
|
||||
def step_given_empty_tag_set(context: Context) -> None:
|
||||
context.tdd_tag_set = set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation execution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("tdd tags I validate the tag set")
|
||||
def step_when_validate(context: Context) -> None:
|
||||
"""Run ``validate_tdd_tags`` and capture any raised ``ValueError``."""
|
||||
context.tdd_validation_error = None
|
||||
try:
|
||||
validate_tdd_tags(context.tdd_tag_set)
|
||||
except ValueError as exc:
|
||||
context.tdd_validation_error = str(exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation outcome assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("tdd tags validation should pass")
|
||||
def step_then_validation_passes(context: Context) -> None:
|
||||
assert context.tdd_validation_error is None, (
|
||||
f"Expected validation to pass but got error: {context.tdd_validation_error}"
|
||||
)
|
||||
|
||||
|
||||
@then('tdd tags validation should fail with error containing "{fragment}"')
|
||||
def step_then_validation_fails_with(context: Context, fragment: str) -> None:
|
||||
assert context.tdd_validation_error is not None, (
|
||||
"Expected validation to fail but it passed"
|
||||
)
|
||||
assert fragment in context.tdd_validation_error, (
|
||||
f"Expected error to contain '{fragment}' but got: "
|
||||
f"{context.tdd_validation_error}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# should_invert_result assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("tdd tags should_invert_result should return true")
|
||||
def step_then_invert_true(context: Context) -> None:
|
||||
assert should_invert_result(context.tdd_tag_set) is True
|
||||
|
||||
|
||||
@then("tdd tags should_invert_result should return false")
|
||||
def step_then_invert_false(context: Context) -> None:
|
||||
assert should_invert_result(context.tdd_tag_set) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion unit-test helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_scenario(
|
||||
tags: list[str],
|
||||
steps_passed: bool = True,
|
||||
hook_failed: bool = False,
|
||||
|
hurui200320
commented
Documentation inaccuracy — This docstring says "The @tdd_expected_fail tag on the scenario causes the after_scenario hook to invert this failure into a pass." This should reference the **Documentation inaccuracy** — This docstring says *"The @tdd_expected_fail tag on the scenario causes the after_scenario hook to invert this failure into a pass."*
This should reference the `Scenario.run()` wrapper, not `after_scenario`. Suggested fix:
```python
"""This step deliberately fails to simulate a bug still being present.
The ``@tdd_expected_fail`` tag on the scenario causes the Scenario.run()
wrapper (installed by _install_tdd_expected_fail_patch) to invert this
failure into a pass.
"""
```
hurui200320
commented
Trivial — Inaccurate docstring This says "the after_scenario hook" but the inversion is actually done in the Should read: "The **Trivial — Inaccurate docstring**
This says "the after_scenario hook" but the inversion is actually done in the `Scenario.run()` monkey-patch (`_install_tdd_expected_fail_patch`), not in `after_scenario`. The `environment.py` code itself correctly documents this at line 560-563.
Should read: "The `@tdd_expected_fail` tag on the scenario causes the `Scenario.run()` wrapper to invert this failure into a pass."
|
||||
was_dry_run: bool = False,
|
||||
step_exception: BaseException | None = None,
|
||||
) -> MagicMock:
|
||||
"""Build a lightweight mock ``Scenario`` for ``apply_tdd_inversion`` tests.
|
||||
|
||||
Args:
|
||||
tags: The effective tags to place on the scenario.
|
||||
steps_passed: When ``True`` every step has ``Status.passed``.
|
||||
When ``False`` the first step has ``Status.failed`` with
|
||||
*step_exception* (defaulting to ``AssertionError``).
|
||||
hook_failed: Simulates an infrastructure/hook error.
|
||||
was_dry_run: Simulates ``--dry-run`` mode.
|
||||
step_exception: The exception attached to the failed step when
|
||||
*steps_passed* is ``False``. Defaults to ``AssertionError``.
|
||||
"""
|
||||
scenario = MagicMock()
|
||||
|
brent.edwards
commented
P2:should-fix (F4) — These 3 "tdd demo" steps (lines ~120-140) are used only by **P2:should-fix (F4)** — These 3 "tdd demo" steps (lines ~120-140) are used only by `tdd_expected_fail_demo.feature`. Per CONTRIBUTING.md §1172-1174, they should live in `features/steps/tdd_expected_fail_demo_steps.py`, not here.
|
||||
scenario.effective_tags = tags
|
||||
scenario.name = "mock-scenario"
|
||||
scenario.hook_failed = hook_failed
|
||||
scenario.was_dry_run = was_dry_run
|
||||
|
||||
mock_step = MagicMock()
|
||||
if steps_passed:
|
||||
mock_step.status = Status.passed
|
||||
mock_step.exception = None
|
||||
else:
|
||||
mock_step.status = Status.failed
|
||||
mock_step.exception = (
|
||||
step_exception
|
||||
if step_exception is not None
|
||||
else AssertionError("simulated assertion failure")
|
||||
)
|
||||
scenario.all_steps = [mock_step]
|
||||
return scenario
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — expected failure inverted to pass (happy path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given(
|
||||
"tdd inversion a mock scenario that fails with AssertionError"
|
||||
" and tdd_expected_fail tags"
|
||||
)
|
||||
def step_given_expected_failure_scenario(context: Context) -> None:
|
||||
"""Create a mock scenario with a failed step (AssertionError) and TDD tags."""
|
||||
context.tdd_mock_scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
step_exception=AssertionError("simulated bug assertion failure"),
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
"tdd inversion the result should be False indicating expected failure was inverted"
|
||||
)
|
||||
def step_then_result_inverted_to_pass(context: Context) -> None:
|
||||
assert context.tdd_inversion_result is False, (
|
||||
f"Expected False (failure inverted to pass) but got "
|
||||
f"{context.tdd_inversion_result}"
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion the scenario status should be passed")
|
||||
def step_then_scenario_status_passed(context: Context) -> None:
|
||||
context.tdd_mock_scenario.set_status.assert_called_with(Status.passed)
|
||||
|
||||
|
||||
@then("tdd inversion all failed and skipped steps should be reset to passed")
|
||||
def step_then_steps_reset_to_passed(context: Context) -> None:
|
||||
for step in context.tdd_mock_scenario.all_steps:
|
||||
assert step.status == Status.passed, (
|
||||
f"Expected step status to be passed but got {step.status}"
|
||||
)
|
||||
assert step.exception is None, (
|
||||
f"Expected step exception to be None but got {step.exception}"
|
||||
)
|
||||
assert step.exc_traceback is None, (
|
||||
f"Expected step exc_traceback to be None but got {step.exc_traceback}"
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion all step error_messages should be cleared")
|
||||
def step_then_error_messages_cleared(context: Context) -> None:
|
||||
"""Assert every step's ``error_message`` is ``None`` after inversion."""
|
||||
for step in context.tdd_mock_scenario.all_steps:
|
||||
assert step.error_message is None, (
|
||||
f"Expected step error_message to be None but got {step.error_message!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — unexpected-pass path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("tdd inversion a mock scenario that passes with tdd_expected_fail tags")
|
||||
def step_given_unexpected_pass_scenario(context: Context) -> None:
|
||||
"""Create a mock scenario where steps pass but @tdd_expected_fail is set."""
|
||||
context.tdd_mock_scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"],
|
||||
steps_passed=True,
|
||||
)
|
||||
|
||||
|
||||
@when("tdd inversion apply_tdd_inversion is called with failed False")
|
||||
def step_when_apply_inversion_not_failed(context: Context) -> None:
|
||||
context.tdd_inversion_result = apply_tdd_inversion(
|
||||
context.tdd_mock_scenario, failed=False
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion the result should be True indicating forced failure")
|
||||
def step_then_result_is_forced_failure(context: Context) -> None:
|
||||
assert context.tdd_inversion_result is True, (
|
||||
f"Expected True (forced failure) but got {context.tdd_inversion_result}"
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion the scenario status should be failed")
|
||||
def step_then_scenario_status_failed(context: Context) -> None:
|
||||
context.tdd_mock_scenario.set_status.assert_called_with(Status.failed)
|
||||
|
||||
|
||||
@then("tdd inversion the last step should have a synthetic error message")
|
||||
def step_then_last_step_has_error(context: Context) -> None:
|
||||
last_step = context.tdd_mock_scenario.all_steps[-1]
|
||||
assert last_step.status == Status.failed, (
|
||||
f"Expected last step status to be failed but got {last_step.status}"
|
||||
)
|
||||
assert isinstance(last_step.exception, AssertionError), (
|
||||
f"Expected AssertionError but got {type(last_step.exception)}"
|
||||
)
|
||||
assert "Bug appears to be fixed" in str(last_step.exception)
|
||||
|
||||
|
||||
@then("tdd inversion the last step should have a synthetic error_message text")
|
||||
def step_then_last_step_has_error_message(context: Context) -> None:
|
||||
"""Assert the last step's ``error_message`` contains the unexpected-pass text."""
|
||||
last_step = context.tdd_mock_scenario.all_steps[-1]
|
||||
assert last_step.error_message is not None, (
|
||||
"Expected last step to have an error_message set, got None"
|
||||
)
|
||||
assert "Bug appears to be fixed" in last_step.error_message, (
|
||||
f"Expected error_message to contain 'Bug appears to be fixed', "
|
||||
f"got {last_step.error_message!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — hook-error guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("tdd inversion a mock scenario with hook_failed and tdd_expected_fail tags")
|
||||
def step_given_hook_failed_scenario(context: Context) -> None:
|
||||
context.tdd_mock_scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
hook_failed=True,
|
||||
)
|
||||
|
||||
|
||||
@when("tdd inversion apply_tdd_inversion is called with failed True")
|
||||
def step_when_apply_inversion_failed(context: Context) -> None:
|
||||
context.tdd_inversion_result = apply_tdd_inversion(
|
||||
context.tdd_mock_scenario, failed=True
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion the result should be True indicating failure was not inverted")
|
||||
def step_then_result_not_inverted(context: Context) -> None:
|
||||
assert context.tdd_inversion_result is True, (
|
||||
f"Expected True (not inverted) but got {context.tdd_inversion_result}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — dry-run guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("tdd inversion a mock scenario in dry-run mode with tdd_expected_fail tags")
|
||||
def step_given_dry_run_scenario(context: Context) -> None:
|
||||
context.tdd_mock_scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"],
|
||||
steps_passed=True,
|
||||
was_dry_run=True,
|
||||
)
|
||||
|
||||
|
||||
@then("tdd inversion the result should be False indicating no inversion occurred")
|
||||
def step_then_result_no_inversion(context: Context) -> None:
|
||||
assert context.tdd_inversion_result is False, (
|
||||
f"Expected False (no inversion) but got {context.tdd_inversion_result}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — non-assertion exception guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("tdd inversion a mock scenario with a RuntimeError and tdd_expected_fail tags")
|
||||
def step_given_non_assertion_scenario(context: Context) -> None:
|
||||
context.tdd_mock_scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_998", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
step_exception=RuntimeError("connection lost"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_tdd_inversion — empty steps edge case
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("tdd inversion a mock scenario with no steps and tdd_expected_fail tags")
|
||||
def step_given_no_steps_scenario(context: Context) -> None:
|
||||
"""Create a mock scenario with no steps and @tdd_expected_fail tags."""
|
||||
scenario = MagicMock()
|
||||
scenario.effective_tags = ["tdd_bug", "tdd_bug_998", "tdd_expected_fail"]
|
||||
scenario.name = "mock-scenario-no-steps"
|
||||
scenario.hook_failed = False
|
||||
scenario.was_dry_run = False
|
||||
scenario.all_steps = []
|
||||
context.tdd_mock_scenario = scenario
|
||||
@@ -1,7 +1,57 @@
|
||||
@infrastructure
|
||||
Feature: TDD expected-fail handler infrastructure
|
||||
Verify that the ``_handle_tdd_expected_fail`` hook in ``environment.py``
|
||||
correctly inverts scenario AND step-level status for TDD bug-capture tests.
|
||||
Verify that ``apply_tdd_inversion`` (the production code path called by the
|
||||
``Scenario.run()`` monkey-patch) and ``handle_tdd_expected_fail`` (the
|
||||
standalone entry point) correctly invert scenario AND step-level status for
|
||||
TDD bug-capture tests.
|
||||
|
||||
# --- apply_tdd_inversion (production path) ---
|
||||
|
||||
Scenario: apply_tdd_inversion inverts a failed scenario with failed and skipped steps to passed
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "broken step" with status "failed"
|
||||
And the mock scenario has a step "skipped step" with status "skipped"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "passed"
|
||||
And the step "broken step" should have status "passed"
|
||||
And the step "skipped step" should have status "passed"
|
||||
And the step "broken step" should have error_message cleared
|
||||
And the step "skipped step" should have error_message cleared
|
||||
|
||||
Scenario: apply_tdd_inversion fails a passed scenario that still carries @tdd_expected_fail
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed"
|
||||
And the mock scenario has a step "last step" with status "passed"
|
||||
When apply_tdd_inversion processes the scenario with failed False
|
||||
Then the scenario status should be "failed"
|
||||
And the step "last step" should have a synthetic error_message
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert when hook_failed is set
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed" and hook_failed
|
||||
And the mock scenario has a step "broken step" with status "failed"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "failed"
|
||||
And the step "broken step" should have status "failed"
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert during dry-run mode
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed" and was_dry_run
|
||||
When apply_tdd_inversion processes the scenario with failed False
|
||||
Then the apply_tdd_inversion result should be False
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert non-AssertionError exceptions
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "failed"
|
||||
And the step "infra step" should have status "failed"
|
||||
|
||||
Scenario: apply_tdd_inversion ignores scenarios without @tdd_expected_fail
|
||||
Given a mock scenario tagged "@tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "broken step" with status "failed"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "failed"
|
||||
And the step "broken step" should have status "failed"
|
||||
|
||||
# --- handle_tdd_expected_fail (standalone entry point) ---
|
||||
|
||||
Scenario: Handler inverts a failed scenario with failed and skipped steps to passed
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
@@ -11,11 +61,15 @@ Feature: TDD expected-fail handler infrastructure
|
||||
Then the scenario status should be "passed"
|
||||
And the step "broken step" should have status "passed"
|
||||
And the step "skipped step" should have status "passed"
|
||||
And the step "broken step" should have error_message cleared
|
||||
And the step "skipped step" should have error_message cleared
|
||||
|
||||
Scenario: Handler fails a passed scenario that still carries @tdd_expected_fail
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "passed"
|
||||
And the mock scenario has a step "last step" with status "passed"
|
||||
When the TDD expected-fail handler processes the scenario
|
||||
Then the scenario status should be "failed"
|
||||
And the step "last step" should have a synthetic error_message
|
||||
|
||||
Scenario: Handler rejects @tdd_expected_fail without @tdd_bug
|
||||
Given a mock scenario tagged "@tdd_expected_fail" with status "failed"
|
||||
@@ -36,3 +90,76 @@ Feature: TDD expected-fail handler infrastructure
|
||||
Given a mock scenario tagged "@tdd_bug @tdd_bug_999" with status "failed"
|
||||
When the TDD expected-fail handler processes the scenario
|
||||
Then the scenario status should be "failed"
|
||||
|
||||
Scenario: Handler does not invert non-AssertionError exceptions
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError"
|
||||
When the TDD expected-fail handler processes the scenario
|
||||
Then the scenario status should be "failed"
|
||||
And the step "infra step" should have status "failed"
|
||||
|
||||
# --- apply_tdd_inversion: mixed exception types (S3) ---
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert when mixed exceptions present
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "bug step" with status "failed"
|
||||
And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "failed"
|
||||
And the step "bug step" should have status "failed"
|
||||
And the step "infra step" should have status "failed"
|
||||
|
||||
Scenario: Handler does not invert when mixed exceptions present
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "bug step" with status "failed"
|
||||
And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError"
|
||||
When the TDD expected-fail handler processes the scenario
|
||||
Then the scenario status should be "failed"
|
||||
And the step "bug step" should have status "failed"
|
||||
And the step "infra step" should have status "failed"
|
||||
|
||||
# --- selective step reset preserves already-passed steps (TC-3) ---
|
||||
|
||||
Scenario: apply_tdd_inversion preserves already-passed steps during inversion
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "ok step" with status "passed"
|
||||
And the mock scenario has a step "broken step" with status "failed"
|
||||
And the mock scenario has a step "skipped step" with status "skipped"
|
||||
When apply_tdd_inversion processes the scenario with failed True
|
||||
Then the scenario status should be "passed"
|
||||
And the step "ok step" should have status "passed"
|
||||
And the step "broken step" should have status "passed"
|
||||
And the step "skipped step" should have status "passed"
|
||||
And the step "broken step" should have error_message cleared
|
||||
And the step "skipped step" should have error_message cleared
|
||||
|
||||
Scenario: Handler preserves already-passed steps during inversion
|
||||
Given a mock scenario tagged "@tdd_expected_fail @tdd_bug @tdd_bug_999" with status "failed"
|
||||
And the mock scenario has a step "ok step" with status "passed"
|
||||
And the mock scenario has a step "broken step" with status "failed"
|
||||
And the mock scenario has a step "skipped step" with status "skipped"
|
||||
When the TDD expected-fail handler processes the scenario
|
||||
Then the scenario status should be "passed"
|
||||
And the step "ok step" should have status "passed"
|
||||
And the step "broken step" should have status "passed"
|
||||
And the step "skipped step" should have status "passed"
|
||||
And the step "broken step" should have error_message cleared
|
||||
And the step "skipped step" should have error_message cleared
|
||||
|
||||
# --- before_scenario hook_failed regression test (TC-2) ---
|
||||
|
||||
Scenario: before_scenario sets hook_failed on invalid TDD tags
|
||||
Given a mock scenario tagged "@tdd_expected_fail" with status "untested"
|
||||
When before_scenario is called with the mock scenario
|
||||
Then the scenario hook_failed flag should be True
|
||||
And the scenario status should be "failed"
|
||||
|
||||
# --- _install_tdd_expected_fail_patch (S2) ---
|
||||
|
||||
Scenario: _install_tdd_expected_fail_patch sets the patched flag
|
||||
When the TDD expected-fail patch installation status is checked
|
||||
Then the Scenario class should have _tdd_run_patched set to True
|
||||
|
||||
Scenario: _install_tdd_expected_fail_patch is idempotent
|
||||
When _install_tdd_expected_fail_patch is called twice
|
||||
Then the Scenario.run method should not be double-wrapped
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
@mock_only
|
||||
Feature: TDD expected-fail result inversion demo
|
||||
Demonstrates the @tdd_expected_fail tag inverting a deliberately failing
|
||||
scenario so it is reported as passed. This exercises the Scenario.run()
|
||||
|
hurui200320
commented
Documentation inaccuracy — This description says "after_scenario hook logic" but the inversion is performed by the **Documentation inaccuracy** — This description says *"after_scenario hook logic"* but the inversion is performed by the `Scenario.run()` monkey-patch installed in `_install_tdd_expected_fail_patch()`, not `after_scenario`. The comment at `environment.py:560` explicitly says the opposite. Please update to:
```
Demonstrates the @tdd_expected_fail tag inverting a deliberately failing
scenario so it is reported as passed. This exercises the Scenario.run()
wrapper installed by _install_tdd_expected_fail_patch() in features/environment.py.
```
|
||||
wrapper logic installed by _install_tdd_expected_fail_patch() in
|
||||
features/environment.py.
|
||||
See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification.
|
||||
|
||||
Note: The "unexpected pass" path (where a @tdd_expected_fail scenario
|
||||
passes and is forced to fail) cannot be tested as an integration scenario
|
||||
because the forced failure would break the test suite. That path is
|
||||
tested via mock-based unit tests in tdd_tag_validation.feature and via
|
||||
real Scenario objects in tdd_expected_fail_infrastructure.feature.
|
||||
|
||||
@tdd_bug @tdd_bug_999 @tdd_expected_fail
|
||||
Scenario: Demo bug 999 expected failure is inverted to pass
|
||||
Given tdd demo a step that always succeeds
|
||||
When tdd demo a deliberately failing assertion is executed
|
||||
Then tdd demo this step is never reached
|
||||
@@ -0,0 +1,126 @@
|
||||
@mock_only
|
||||
Feature: TDD bug-capture tag validation
|
||||
As a developer writing TDD bug-capture tests
|
||||
I want the environment hooks to validate my tag combinations
|
||||
So that misconfigured TDD tests are caught before they run
|
||||
|
||||
# Tests for the validate_tdd_tags() and should_invert_result() helper
|
||||
# functions defined in features/environment.py.
|
||||
# See CONTRIBUTING.md > TDD Bug Test Tags for the three-tag specification.
|
||||
|
||||
# --- validate_tdd_tags: valid combinations ---
|
||||
|
||||
Scenario: Valid TDD tags with all three tags present
|
||||
Given tdd tags a tag set with "tdd_bug, tdd_bug_123, tdd_expected_fail"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should pass
|
||||
|
||||
Scenario: Valid TDD tags with tdd_bug and tdd_bug_N only
|
||||
Given tdd tags a tag set with "tdd_bug, tdd_bug_456"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should pass
|
||||
|
||||
Scenario: Valid TDD tags with tdd_bug only
|
||||
Given tdd tags a tag set with "tdd_bug"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should pass
|
||||
|
||||
Scenario: Valid TDD tags with no TDD tags at all
|
||||
Given tdd tags a tag set with "wip, slow"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should pass
|
||||
|
||||
Scenario: Valid TDD tags with empty tag set
|
||||
Given tdd tags an empty tag set
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should pass
|
||||
|
||||
# --- validate_tdd_tags: invalid combinations ---
|
||||
|
||||
Scenario: Invalid TDD tags tdd_bug_N without tdd_bug raises error
|
||||
Given tdd tags a tag set with "tdd_bug_123"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should fail with error containing "@tdd_bug"
|
||||
|
||||
Scenario: Invalid TDD tags tdd_expected_fail without tdd_bug raises error
|
||||
Given tdd tags a tag set with "tdd_expected_fail, tdd_bug_123"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should fail with error containing "@tdd_bug"
|
||||
|
||||
Scenario: Invalid TDD tags tdd_expected_fail without tdd_bug_N raises error
|
||||
Given tdd tags a tag set with "tdd_expected_fail, tdd_bug"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should fail with error containing "@tdd_bug_<N>"
|
||||
|
||||
Scenario: Invalid TDD tags tdd_expected_fail alone raises error
|
||||
Given tdd tags a tag set with "tdd_expected_fail"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should fail with error containing "@tdd_bug"
|
||||
|
||||
Scenario: Invalid TDD tags multiple tdd_bug_N without tdd_bug raises error
|
||||
Given tdd tags a tag set with "tdd_bug_100, tdd_bug_200"
|
||||
When tdd tags I validate the tag set
|
||||
Then tdd tags validation should fail with error containing "@tdd_bug"
|
||||
|
||||
# --- should_invert_result ---
|
||||
|
||||
Scenario: should_invert_result returns True when tdd_expected_fail is present
|
||||
Given tdd tags a tag set with "tdd_bug, tdd_bug_42, tdd_expected_fail"
|
||||
Then tdd tags should_invert_result should return true
|
||||
|
||||
Scenario: should_invert_result returns False when tdd_expected_fail is absent
|
||||
Given tdd tags a tag set with "tdd_bug, tdd_bug_42"
|
||||
Then tdd tags should_invert_result should return false
|
||||
|
||||
Scenario: should_invert_result returns False for empty tag set
|
||||
Given tdd tags an empty tag set
|
||||
Then tdd tags should_invert_result should return false
|
||||
|
||||
# --- apply_tdd_inversion: expected failure inverted to pass (happy path) ---
|
||||
|
||||
Scenario: apply_tdd_inversion inverts expected AssertionError failure to pass
|
||||
Given tdd inversion a mock scenario that fails with AssertionError and tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed True
|
||||
Then tdd inversion the result should be False indicating expected failure was inverted
|
||||
And tdd inversion the scenario status should be passed
|
||||
And tdd inversion all failed and skipped steps should be reset to passed
|
||||
And tdd inversion all step error_messages should be cleared
|
||||
|
||||
# --- apply_tdd_inversion: unexpected pass forces failure ---
|
||||
|
||||
Scenario: apply_tdd_inversion forces failure when tdd_expected_fail scenario passes
|
||||
Given tdd inversion a mock scenario that passes with tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed False
|
||||
Then tdd inversion the result should be True indicating forced failure
|
||||
And tdd inversion the scenario status should be failed
|
||||
And tdd inversion the last step should have a synthetic error message
|
||||
And tdd inversion the last step should have a synthetic error_message text
|
||||
|
||||
# --- apply_tdd_inversion: hook-error guard ---
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert when hook_failed is True
|
||||
Given tdd inversion a mock scenario with hook_failed and tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed True
|
||||
Then tdd inversion the result should be True indicating failure was not inverted
|
||||
|
||||
# --- apply_tdd_inversion: dry-run guard ---
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert during dry-run mode
|
||||
Given tdd inversion a mock scenario in dry-run mode with tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed False
|
||||
Then tdd inversion the result should be False indicating no inversion occurred
|
||||
|
||||
# --- apply_tdd_inversion: non-assertion exception guard ---
|
||||
|
||||
Scenario: apply_tdd_inversion does not invert non-AssertionError exceptions
|
||||
Given tdd inversion a mock scenario with a RuntimeError and tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed True
|
||||
Then tdd inversion the result should be True indicating failure was not inverted
|
||||
|
||||
# --- apply_tdd_inversion: empty steps edge case ---
|
||||
|
||||
Scenario: apply_tdd_inversion handles unexpected pass with no steps
|
||||
Given tdd inversion a mock scenario with no steps and tdd_expected_fail tags
|
||||
When tdd inversion apply_tdd_inversion is called with failed False
|
||||
Then tdd inversion the result should be True indicating forced failure
|
||||
And tdd inversion the scenario status should be failed
|
||||
@@ -0,0 +1,283 @@
|
||||
"""Helper script for tdd_tag_validation.robot integration tests.
|
||||
|
||||
Each subcommand is a self-contained check that exercises the TDD
|
||||
bug-capture tag validation and result-inversion logic defined in
|
||||
``features/environment.py``. Sentinels are printed on success so
|
||||
the Robot test case can assert correct behaviour.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Ensure the project root is importable so ``features.environment`` resolves.
|
||||
_ROOT = str(Path(__file__).resolve().parents[1])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from behave.model import Status # noqa: E402
|
||||
from features.environment import ( # noqa: E402
|
||||
_UNEXPECTED_PASS_MSG,
|
||||
apply_tdd_inversion,
|
||||
should_invert_result,
|
||||
validate_tdd_tags,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_mock_scenario(
|
||||
tags: list[str],
|
||||
steps_passed: bool = True,
|
||||
hook_failed: bool = False,
|
||||
was_dry_run: bool = False,
|
||||
step_exception: BaseException | None = None,
|
||||
) -> MagicMock:
|
||||
"""Build a lightweight mock ``Scenario`` for ``apply_tdd_inversion``."""
|
||||
scenario = MagicMock()
|
||||
scenario.effective_tags = tags
|
||||
scenario.name = "mock-scenario"
|
||||
scenario.hook_failed = hook_failed
|
||||
scenario.was_dry_run = was_dry_run
|
||||
|
||||
mock_step = MagicMock()
|
||||
if steps_passed:
|
||||
mock_step.status = Status.passed
|
||||
mock_step.exception = None
|
||||
else:
|
||||
mock_step.status = Status.failed
|
||||
mock_step.exception = (
|
||||
step_exception
|
||||
if step_exception is not None
|
||||
else AssertionError("simulated assertion failure")
|
||||
)
|
||||
scenario.all_steps = [mock_step]
|
||||
return scenario
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands — validate_tdd_tags
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def validate_tags_valid_combos() -> None:
|
||||
"""Verify valid tag combinations raise no error."""
|
||||
valid_sets: list[set[str]] = [
|
||||
{"tdd_bug", "tdd_bug_42"},
|
||||
{"tdd_bug", "tdd_bug_42", "tdd_expected_fail"},
|
||||
{"tdd_bug"},
|
||||
{"some_other_tag"},
|
||||
set(),
|
||||
]
|
||||
for tag_set in valid_sets:
|
||||
try:
|
||||
validate_tdd_tags(tag_set)
|
||||
except ValueError as exc:
|
||||
print(
|
||||
f"FAIL: valid tag set {tag_set} raised ValueError: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
print("validate-tags-valid-combos-ok")
|
||||
|
||||
|
||||
def validate_tags_bug_n_without_bug() -> None:
|
||||
"""Verify @tdd_bug_<N> without @tdd_bug raises ValueError."""
|
||||
try:
|
||||
validate_tdd_tags({"tdd_bug_42"})
|
||||
except ValueError:
|
||||
print("validate-tags-bug-n-without-bug-ok")
|
||||
return
|
||||
print("FAIL: expected ValueError for tdd_bug_42 without tdd_bug", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validate_tags_expected_fail_missing_bug() -> None:
|
||||
"""Verify @tdd_expected_fail without @tdd_bug raises ValueError."""
|
||||
try:
|
||||
validate_tdd_tags({"tdd_expected_fail", "tdd_bug_42"})
|
||||
except ValueError:
|
||||
print("validate-tags-expected-fail-missing-bug-ok")
|
||||
return
|
||||
print(
|
||||
"FAIL: expected ValueError for tdd_expected_fail without tdd_bug",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validate_tags_expected_fail_missing_bug_n() -> None:
|
||||
"""Verify @tdd_expected_fail without @tdd_bug_<N> raises ValueError."""
|
||||
try:
|
||||
validate_tdd_tags({"tdd_expected_fail", "tdd_bug"})
|
||||
except ValueError:
|
||||
print("validate-tags-expected-fail-missing-bug-n-ok")
|
||||
return
|
||||
print(
|
||||
"FAIL: expected ValueError for tdd_expected_fail without tdd_bug_<N>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands — should_invert_result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def should_invert_with_expected_fail() -> None:
|
||||
"""Verify should_invert_result returns True when @tdd_expected_fail present."""
|
||||
result = should_invert_result({"tdd_bug", "tdd_bug_42", "tdd_expected_fail"})
|
||||
if result is not True:
|
||||
print(f"FAIL: expected True, got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("should-invert-with-expected-fail-ok")
|
||||
|
||||
|
||||
def should_not_invert_without_expected_fail() -> None:
|
||||
"""Verify should_invert_result returns False without @tdd_expected_fail."""
|
||||
result = should_invert_result({"tdd_bug", "tdd_bug_42"})
|
||||
if result is not False:
|
||||
print(f"FAIL: expected False, got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("should-not-invert-without-expected-fail-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands — apply_tdd_inversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def inversion_expected_fail() -> None:
|
||||
"""Verify expected failure (failed=True) is inverted to passed."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=True)
|
||||
if result is not False:
|
||||
print(f"FAIL: expected False (inverted), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
scenario.set_status.assert_called_with(Status.passed)
|
||||
print("inversion-expected-fail-ok")
|
||||
|
||||
|
||||
def inversion_unexpected_pass() -> None:
|
||||
"""Verify unexpected pass (failed=False) is inverted to failure."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"],
|
||||
steps_passed=True,
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=False)
|
||||
if result is not True:
|
||||
print(f"FAIL: expected True (forced failure), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
scenario.set_status.assert_called_with(Status.failed)
|
||||
# Verify synthetic error is attached to last step
|
||||
last_step = scenario.all_steps[-1]
|
||||
if last_step.status != Status.failed:
|
||||
print(
|
||||
f"FAIL: last step status should be failed, got {last_step.status}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
if not isinstance(last_step.exception, AssertionError):
|
||||
print(
|
||||
f"FAIL: expected AssertionError, got {type(last_step.exception)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
if str(last_step.exception) != _UNEXPECTED_PASS_MSG:
|
||||
print(f"FAIL: unexpected message: {last_step.exception}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("inversion-unexpected-pass-ok")
|
||||
|
||||
|
||||
def inversion_hook_error_guard() -> None:
|
||||
"""Verify hook_failed prevents inversion."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
hook_failed=True,
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=True)
|
||||
if result is not True:
|
||||
print(f"FAIL: expected True (not inverted), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("inversion-hook-error-guard-ok")
|
||||
|
||||
|
||||
def inversion_dry_run_guard() -> None:
|
||||
"""Verify dry-run mode prevents inversion."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"],
|
||||
steps_passed=True,
|
||||
was_dry_run=True,
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=False)
|
||||
if result is not False:
|
||||
print(f"FAIL: expected False (not inverted), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("inversion-dry-run-guard-ok")
|
||||
|
||||
|
||||
def inversion_non_assertion_guard() -> None:
|
||||
"""Verify non-AssertionError exceptions prevent inversion."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42", "tdd_expected_fail"],
|
||||
steps_passed=False,
|
||||
step_exception=RuntimeError("connection lost"),
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=True)
|
||||
if result is not True:
|
||||
print(f"FAIL: expected True (not inverted), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("inversion-non-assertion-guard-ok")
|
||||
|
||||
|
||||
def inversion_no_tag_passthrough() -> None:
|
||||
"""Verify scenarios without @tdd_expected_fail are not modified."""
|
||||
scenario = _make_mock_scenario(
|
||||
tags=["tdd_bug", "tdd_bug_42"],
|
||||
steps_passed=False,
|
||||
)
|
||||
result = apply_tdd_inversion(scenario, failed=True)
|
||||
if result is not True:
|
||||
print(f"FAIL: expected True (passthrough), got {result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# set_status should NOT have been called
|
||||
scenario.set_status.assert_not_called()
|
||||
print("inversion-no-tag-passthrough-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS = {
|
||||
"validate_tags_valid_combos": validate_tags_valid_combos,
|
||||
"validate_tags_bug_n_without_bug": validate_tags_bug_n_without_bug,
|
||||
"validate_tags_expected_fail_missing_bug": validate_tags_expected_fail_missing_bug,
|
||||
"validate_tags_expected_fail_missing_bug_n": (
|
||||
validate_tags_expected_fail_missing_bug_n
|
||||
),
|
||||
"should_invert_with_expected_fail": should_invert_with_expected_fail,
|
||||
"should_not_invert_without_expected_fail": should_not_invert_without_expected_fail,
|
||||
"inversion_expected_fail": inversion_expected_fail,
|
||||
"inversion_unexpected_pass": inversion_unexpected_pass,
|
||||
"inversion_hook_error_guard": inversion_hook_error_guard,
|
||||
"inversion_dry_run_guard": inversion_dry_run_guard,
|
||||
"inversion_non_assertion_guard": inversion_non_assertion_guard,
|
||||
"inversion_no_tag_passthrough": inversion_no_tag_passthrough,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <command>", file=sys.stderr)
|
||||
print(f"Commands: {list(_COMMANDS)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
_COMMANDS[sys.argv[1]]()
|
||||
@@ -0,0 +1,134 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for TDD bug-capture tag validation and
|
||||
... @tdd_expected_fail result inversion logic defined in
|
||||
... features/environment.py. Covers validate_tdd_tags(),
|
||||
... should_invert_result(), and apply_tdd_inversion() including
|
||||
... all guard paths (hook errors, dry-run, non-assertion exceptions).
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_tdd_tag_validation.py
|
||||
|
||||
*** Test Cases ***
|
||||
Valid Tag Combinations Are Accepted
|
||||
[Documentation] Verify valid TDD tag sets raise no errors
|
||||
[Tags] testing tdd validation
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validate_tags_valid_combos
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} validate-tags-valid-combos-ok
|
||||
|
||||
Bug N Without Bug Tag Raises Error
|
||||
[Documentation] Verify @tdd_bug_<N> without @tdd_bug raises ValueError
|
||||
[Tags] testing tdd validation
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validate_tags_bug_n_without_bug
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} validate-tags-bug-n-without-bug-ok
|
||||
|
||||
Expected Fail Without Bug Tag Raises Error
|
||||
[Documentation] Verify @tdd_expected_fail without @tdd_bug raises ValueError
|
||||
[Tags] testing tdd validation
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validate_tags_expected_fail_missing_bug
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} validate-tags-expected-fail-missing-bug-ok
|
||||
|
||||
Expected Fail Without Bug N Tag Raises Error
|
||||
[Documentation] Verify @tdd_expected_fail without @tdd_bug_<N> raises ValueError
|
||||
[Tags] testing tdd validation
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validate_tags_expected_fail_missing_bug_n
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} validate-tags-expected-fail-missing-bug-n-ok
|
||||
|
||||
Should Invert With Expected Fail Tag
|
||||
[Documentation] Verify should_invert_result returns True with @tdd_expected_fail
|
||||
[Tags] testing tdd inversion
|
||||
${result}= Run Process ${PYTHON} ${HELPER} should_invert_with_expected_fail
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} should-invert-with-expected-fail-ok
|
||||
|
||||
Should Not Invert Without Expected Fail Tag
|
||||
[Documentation] Verify should_invert_result returns False without @tdd_expected_fail
|
||||
[Tags] testing tdd inversion
|
||||
${result}= Run Process ${PYTHON} ${HELPER} should_not_invert_without_expected_fail
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} should-not-invert-without-expected-fail-ok
|
||||
|
||||
Inversion Expected Fail Is Inverted To Pass
|
||||
[Documentation] Verify a failed @tdd_expected_fail scenario is inverted to passed
|
||||
[Tags] testing tdd inversion
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_expected_fail
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-expected-fail-ok
|
||||
|
||||
Inversion Unexpected Pass Forces Failure
|
||||
[Documentation] Verify a passing @tdd_expected_fail scenario is forced to fail
|
||||
... with a synthetic error attached to the last step
|
||||
[Tags] testing tdd inversion
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_unexpected_pass
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-unexpected-pass-ok
|
||||
|
||||
Inversion Hook Error Guard Prevents Inversion
|
||||
[Documentation] Verify hook_failed=True prevents result inversion
|
||||
[Tags] testing tdd inversion guard
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_hook_error_guard
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-hook-error-guard-ok
|
||||
|
||||
Inversion Dry Run Guard Prevents Inversion
|
||||
[Documentation] Verify dry-run mode prevents result inversion
|
||||
[Tags] testing tdd inversion guard
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_dry_run_guard
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-dry-run-guard-ok
|
||||
|
||||
Inversion Non Assertion Guard Prevents Inversion
|
||||
[Documentation] Verify non-AssertionError exceptions prevent result inversion
|
||||
[Tags] testing tdd inversion guard
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_non_assertion_guard
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-non-assertion-guard-ok
|
||||
|
||||
Inversion No Tag Passthrough
|
||||
[Documentation] Verify scenarios without @tdd_expected_fail are not modified
|
||||
[Tags] testing tdd inversion
|
||||
${result}= Run Process ${PYTHON} ${HELPER} inversion_no_tag_passthrough
|
||||
... cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} inversion-no-tag-passthrough-ok
|
||||
Minor — Comment inaccuracy
This comment says the helpers are "called from the
before_scenarioandafter_scenariohooks respectively" butshould_invert_resultis actually called from_tdd_aware_run(theScenario.run()wrapper installed inbefore_all), not fromafter_scenario. Theafter_scenariohook itself has a NOTE (line 560) clarifying this. Consider updating this comment to match.