diff --git a/CHANGELOG.md b/CHANGELOG.md index 403389fd0..36ca0cc2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **TDD Non-AssertionError Guard Visibility** (#8294): `apply_tdd_inversion` in + `features/environment.py` now emits its non-assertion exception guard warning to + both the structured logger and `stderr` via a new `_warning_with_stderr` helper. + This makes the guard firing visible in standard Behave console output and CI log + snippets where the structured logging sink may not be displayed. BDD infrastructure + coverage added: a new scenario in `tdd_expected_fail_infrastructure.feature` + asserts that the warning is emitted to stderr when a non-AssertionError exception + is encountered in an `@tdd_expected_fail` scenario, and a second scenario asserts + the warning is NOT emitted when the exception is an `AssertionError`. The + `CONTRIBUTING.md` now documents that `@tdd_expected_fail` step definitions must + signal expected failures via `AssertionError`. + - **Parallel Behave Runner Log Noise Reduction** (#8351): The parallel behave runner now suppresses captured stdout/stderr for passing worker chunks and only replays diagnostics for failed, errored, or crashed chunks. This makes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9aa334ad6..700234833 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1218,6 +1218,11 @@ These rules are enforced by the test environment hooks and CI quality gates: - Any scenario with `@tdd_issue_` **must** also have `@tdd_issue`. - Any scenario with `@tdd_expected_fail` **must** also have `@tdd_issue` and at least one `@tdd_issue_`. +- In `@tdd_expected_fail` scenarios, step definitions that intentionally signal "bug still + present" **must fail via `AssertionError`** (for example by using `assert` statements or + `raise AssertionError(...)`). Do not use non-assertion exceptions (such as `ValueError`, + `RuntimeError`, or `OSError`) to represent expected bug failures; those are treated as + infrastructure/runtime errors and are not inverted. - A bug fix PR that closes issue `#N` **must** remove `@tdd_expected_fail` from all scenarios tagged `@tdd_issue_N`. If the tag is still present, the CI quality gate blocks the PR. - A bug fix PR that closes issue `#N` where no `@tdd_issue_N` test exists in the codebase is diff --git a/features/environment.py b/features/environment.py index 0c2c7a3bc..f5690d91b 100644 --- a/features/environment.py +++ b/features/environment.py @@ -72,6 +72,20 @@ _INITIALIZED_DBS: set[str] = set() _tdd_logger = logging.getLogger("cleveragents.testing.tdd_tags") +def _warning_with_stderr(message: str) -> None: + """Emit a TDD warning to both logger output and stderr. + + Behave's standard console output captures stderr reliably, while project + logging configuration may route warning logs to structured sinks that are + not always visible in CI snippets. Emitting to both channels makes + non-inversion guard firings obvious during flaky-test diagnosis. + """ + _tdd_logger.warning(message) + print( + message, file=sys.stderr + ) # Intentional duplication: logger may route to structured sinks not visible in CI; stderr guarantees visibility in Behave output. + + def validate_tdd_tags(tags: set[str]) -> None: """Validate TDD issue-capture tag combinations. @@ -190,12 +204,18 @@ def apply_tdd_inversion(scenario: Any, failed: bool) -> bool: and step.exception is not None and not isinstance(step.exception, AssertionError) ): - _tdd_logger.warning( + # Intentional f-string (eager evaluation) rather than + # lazy %-style: the message is always emitted to stderr via + # print(), so deferred formatting buys nothing here. + exc_text = str( + step.exception + )[ + :500 + ] # Defensive truncation — avoids runaway output for exceptions with very long str() representations. + _warning_with_stderr( "Non-assertion exception in expected-fail scenario " - "'%s' step '%s': %s — not inverting.", - scenario.name, - step.name, - step.exception, + f"'{scenario.name}' step '{step.name}': " + f"{exc_text} — not inverting." ) return failed diff --git a/features/steps/tdd_expected_fail_infrastructure_steps.py b/features/steps/tdd_expected_fail_infrastructure_steps.py index 141fa3d9a..e5d67c536 100644 --- a/features/steps/tdd_expected_fail_infrastructure_steps.py +++ b/features/steps/tdd_expected_fail_infrastructure_steps.py @@ -10,6 +10,9 @@ exceptions). from __future__ import annotations +from contextlib import redirect_stderr +from io import StringIO + from behave import given, then, when from behave.model import Scenario, Status, Step from behave.runner import Context @@ -125,6 +128,19 @@ def step_run_apply_inversion_failed(context: Context) -> None: ) +@when( + "apply_tdd_inversion processes the scenario with failed True while capturing stderr" +) +def step_run_apply_inversion_failed_capture_stderr(context: Context) -> None: + """Invoke ``apply_tdd_inversion`` and capture stderr output.""" + stderr_buffer = StringIO() + with redirect_stderr(stderr_buffer): + context.apply_inversion_result = apply_tdd_inversion( + context.mock_scenario, failed=True + ) + context.captured_stderr = stderr_buffer.getvalue() + + @when("apply_tdd_inversion processes the scenario with failed False") def step_run_apply_inversion_not_failed(context: Context) -> None: """Invoke ``apply_tdd_inversion`` with ``failed=False``.""" @@ -151,6 +167,32 @@ def step_check_inversion_result_true(context: Context) -> None: ) +@then('stderr output should contain "{expected_text}"') +def step_check_stderr_contains(context: Context, expected_text: str) -> None: + """Assert captured stderr output contains the expected text.""" + assert hasattr(context, "captured_stderr"), ( + "context.captured_stderr was not set — use the " + "'while capturing stderr' When step to capture stderr before asserting." + ) + stderr_output: str = context.captured_stderr + assert expected_text in stderr_output, ( + f"Expected stderr to contain {expected_text!r}, got: {stderr_output!r}" + ) + + +@then('stderr output should not contain "{expected_text}"') +def step_check_stderr_not_contains(context: Context, expected_text: str) -> None: + """Assert captured stderr output does not contain the given text.""" + assert hasattr(context, "captured_stderr"), ( + "context.captured_stderr was not set — use the " + "'while capturing stderr' When step to capture stderr before asserting." + ) + stderr_output: str = context.captured_stderr + assert expected_text not in stderr_output, ( + f"Expected stderr NOT to contain {expected_text!r}, got: {stderr_output!r}" + ) + + # --------------------------------------------------------------------------- # handle_tdd_expected_fail steps (standalone entry point) # --------------------------------------------------------------------------- diff --git a/features/tdd_expected_fail_infrastructure.feature b/features/tdd_expected_fail_infrastructure.feature index 17db46aa2..9bb851dd5 100644 --- a/features/tdd_expected_fail_infrastructure.feature +++ b/features/tdd_expected_fail_infrastructure.feature @@ -44,6 +44,24 @@ Feature: TDD expected-fail handler infrastructure Then the scenario status should be "failed" And the step "infra step" should have status "failed" + Scenario: apply_tdd_inversion surfaces non-AssertionError guard warning to stderr + Given a mock scenario tagged "@tdd_expected_fail @tdd_issue @tdd_issue_999" with status "failed" + And the mock scenario has an infrastructure-error step "infra step" with exception "RuntimeError" + When apply_tdd_inversion processes the scenario with failed True while capturing stderr + Then the scenario status should be "failed" + And the step "infra step" should have status "failed" + And the apply_tdd_inversion result should be True + And stderr output should contain "Non-assertion exception in expected-fail scenario" + + Scenario: apply_tdd_inversion does not emit guard warning for AssertionError exceptions + Given a mock scenario tagged "@tdd_expected_fail @tdd_issue @tdd_issue_999" with status "failed" + And the mock scenario has a step "bug step" with status "failed" + When apply_tdd_inversion processes the scenario with failed True while capturing stderr + Then the scenario status should be "passed" + And the step "bug step" should have status "passed" + And the apply_tdd_inversion result should be False + And stderr output should not contain "Non-assertion exception" + Scenario: apply_tdd_inversion ignores scenarios without @tdd_expected_fail Given a mock scenario tagged "@tdd_issue @tdd_issue_999" with status "failed" And the mock scenario has a step "broken step" with status "failed"