diff --git a/features/steps/tdd_robot_listener_guards_steps.py b/features/steps/tdd_robot_listener_guards_steps.py new file mode 100644 index 000000000..db03d2886 --- /dev/null +++ b/features/steps/tdd_robot_listener_guards_steps.py @@ -0,0 +1,162 @@ +"""Behave step implementations for the Robot TDD listener guard tests.""" + +import importlib.util +import sys +from itertools import count +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any + +from behave import given, then, when + +# --------------------------------------------------------------------------- +# Load the listener module from the project's ``robot/`` directory using +# ``spec_from_file_location``. A bare ``importlib.import_module("robot.…")`` +# resolves to the *Robot Framework* package instead of the local directory +# because ``robot/`` has no ``__init__.py``. +# --------------------------------------------------------------------------- + +_ROBOT_DIR = Path(__file__).resolve().parent.parent.parent / "robot" +_spec = importlib.util.spec_from_file_location( + "tdd_expected_fail_listener", + _ROBOT_DIR / "tdd_expected_fail_listener.py", +) +assert _spec is not None and _spec.loader is not None +listener: ModuleType = importlib.util.module_from_spec(_spec) +sys.modules["tdd_expected_fail_listener"] = listener +_spec.loader.exec_module(listener) + +_COUNTER = count() + + +@when('I evaluate the infrastructure guard for message "{message}"') +def step_evaluate_infrastructure_guard(context: Any, message: str) -> None: + context.infrastructure_guard_result = listener._is_infrastructure_error(message) + + +@when("I evaluate the infrastructure guard for an empty message") +def step_evaluate_infrastructure_guard_empty(context: Any) -> None: + """Dedicated step for the empty-string case (Behave parse cannot match ``""`` to ``{message}``).""" + context.infrastructure_guard_result = listener._is_infrastructure_error("") + + +@then("the infrastructure guard result is True") +def step_infrastructure_guard_true(context: Any) -> None: + assert context.infrastructure_guard_result is True, ( + context.infrastructure_guard_result + ) + + +@then("the infrastructure guard result is False") +def step_infrastructure_guard_false(context: Any) -> None: + assert context.infrastructure_guard_result is False, ( + context.infrastructure_guard_result + ) + + +def _build_status_namespace(status: str | None) -> SimpleNamespace | None: + if status is None: + return None + return SimpleNamespace(status=status) + + +@given( + 'a result with setup status "{setup_status}" and teardown status "{teardown_status}" ' + 'and message "{message}"' +) +def step_setup_teardown_result( + context: Any, setup_status: str, teardown_status: str, message: str +) -> None: + def _normalize(raw: str) -> str | None: + return None if raw.upper() == "NONE" else raw + + context.guard_result_input = SimpleNamespace( + setup=_build_status_namespace(_normalize(setup_status)), + teardown=_build_status_namespace(_normalize(teardown_status)), + message=message, + ) + + +@given( + 'a result with setup status "{setup_status}" and teardown status "{teardown_status}" ' + "and an empty message" +) +def step_setup_teardown_result_empty_message( + context: Any, setup_status: str, teardown_status: str +) -> None: + """Dedicated step for the empty-message case (Behave parse cannot match ``""`` to ``{message}``).""" + + def _normalize(raw: str) -> str | None: + return None if raw.upper() == "NONE" else raw + + context.guard_result_input = SimpleNamespace( + setup=_build_status_namespace(_normalize(setup_status)), + teardown=_build_status_namespace(_normalize(teardown_status)), + message="", + ) + + +@when("I evaluate the setup teardown guard") +def step_evaluate_setup_teardown_guard(context: Any) -> None: + context.setup_teardown_guard_result = listener._has_setup_teardown_failure( + context.guard_result_input + ) + + +@then("the setup teardown guard result is True") +def step_assert_setup_teardown_true(context: Any) -> None: + assert context.setup_teardown_guard_result is True, ( + context.setup_teardown_guard_result + ) + + +@then("the setup teardown guard result is False") +def step_assert_setup_teardown_false(context: Any) -> None: + assert context.setup_teardown_guard_result is False, ( + context.setup_teardown_guard_result + ) + + +@given("a tdd listener result with body keyword statuses") +def step_define_result_with_body_statuses(context: Any) -> None: + body_statuses = [row["status"].strip() for row in context.table] + context.result = SimpleNamespace( + full_name=f"behave-listener::{next(_COUNTER)}", + status="PASS", + message="", + setup=None, + teardown=None, + body=[SimpleNamespace(status=status) for status in body_statuses], + ) + context.tags = ["tdd_issue", "tdd_issue_999", "tdd_expected_fail"] + + +@given('the result initial status is "{status}"') +def step_set_initial_status(context: Any, status: str) -> None: + context.result.status = status + if status == "FAIL": + context.result.message = "Original failure" + else: + context.result.message = "" + + +@when('the listener processes the result with tags "{tags}"') +def step_listener_processes_result(context: Any, tags: str) -> None: + listener.close() + tag_list = [tag.strip().lower() for tag in tags.split()] if tags.strip() else [] + data = SimpleNamespace(tags=tag_list) + context.result.full_name = f"behave-listener::{next(_COUNTER)}" + listener.end_test(data, context.result) + context.processed_result = context.result + + +@then('the processed result status is "{expected}"') +def step_assert_processed_status(context: Any, expected: str) -> None: + assert context.processed_result.status == expected, context.processed_result.status + + +@then('the processed result message contains "{fragment}"') +def step_assert_processed_message_contains(context: Any, fragment: str) -> None: + assert fragment in context.processed_result.message, ( + context.processed_result.message + ) diff --git a/features/testing/tdd_robot_listener_guards.feature b/features/testing/tdd_robot_listener_guards.feature new file mode 100644 index 000000000..320e0097a --- /dev/null +++ b/features/testing/tdd_robot_listener_guards.feature @@ -0,0 +1,93 @@ +Feature: Robot TDD listener guard helper behaviour + The pure Python guard helpers backing the Robot TDD listener should behave + deterministically to avoid false positives or negatives. These scenarios + exercise the infrastructure error detection, setup/teardown guard, and + dry-run detection logic with representative inputs. + + Scenario Outline: Infrastructure error patterns are recognised + When I evaluate the infrastructure guard for message "" + Then the infrastructure guard result is True + + Examples: + | message | + | No keyword with name Foo | + | TypeError: bad call | + | ImportError: missing module | + | ModuleNotFoundError: mylib | + | FileNotFoundError: config.json | + | TimeoutError: network stalled | + | PermissionError: denied | + | OSError: device busy | + | ConnectionError: dropped | + | AttributeError: missing attr | + | Connection refused by peer | + | Connection reset by host | + | Connection timed out after 5s | + | No library with name FooLibrary | + | Importing library FooLibrary failed | + + Scenario: Variable pattern matches Robot-style placeholders only + When I evaluate the infrastructure guard for message "Variable '${FOO}' not found" + Then the infrastructure guard result is True + + Scenario: Variable pattern ignores plain quoted values + When I evaluate the infrastructure guard for message "Variable 'foo' not found" + Then the infrastructure guard result is False + + Scenario: Normal assertion failure is not treated as infrastructure + When I evaluate the infrastructure guard for message "AssertionError: expected 1 but got 2" + Then the infrastructure guard result is False + + Scenario: Blank failure message is not treated as infrastructure + When I evaluate the infrastructure guard for an empty message + Then the infrastructure guard result is False + + Scenario: Setup failure flag triggers the setup/teardown guard + Given a result with setup status "FAIL" and teardown status "PASS" and an empty message + When I evaluate the setup teardown guard + Then the setup teardown guard result is True + + Scenario: Teardown failure flag triggers the setup/teardown guard + Given a result with setup status "PASS" and teardown status "FAIL" and an empty message + When I evaluate the setup teardown guard + Then the setup teardown guard result is True + + Scenario: Teardown failure message triggers the fallback detection + Given a result with setup status "NONE" and teardown status "NONE" and message "Teardown failed: cleanup crashed" + When I evaluate the setup teardown guard + Then the setup teardown guard result is True + + Scenario: Setup failure message triggers the fallback detection + Given a result with setup status "NONE" and teardown status "NONE" and message "Setup failed: database unavailable" + When I evaluate the setup teardown guard + Then the setup teardown guard result is True + + Scenario: Setup teardown guard ignores clean results + Given a result with setup status "PASS" and teardown status "PASS" and message "All good" + When I evaluate the setup teardown guard + Then the setup teardown guard result is False + + Scenario: Dry-run guard leaves PASS results unchanged + Given a tdd listener result with body keyword statuses + | status | + | NOT RUN | + | NOT RUN | + And the result initial status is "PASS" + When the listener processes the result with tags "tdd_issue tdd_issue_999 tdd_expected_fail" + Then the processed result status is "PASS" + + Scenario: Dry-run guard allows inversion when keywords executed + Given a tdd listener result with body keyword statuses + | status | + | PASS | + And the result initial status is "FAIL" + When the listener processes the result with tags "tdd_issue tdd_issue_999 tdd_expected_fail" + Then the processed result status is "PASS" + And the processed result message contains "failed as expected" + + Scenario: Empty body does not trigger the dry-run guard + Given a tdd listener result with body keyword statuses + | status | + And the result initial status is "FAIL" + When the listener processes the result with tags "tdd_issue tdd_issue_999 tdd_expected_fail" + Then the processed result status is "PASS" diff --git a/robot/_tdd_fixture_runner.py b/robot/_tdd_fixture_runner.py index 07cb5ddb6..4e2a32d12 100644 --- a/robot/_tdd_fixture_runner.py +++ b/robot/_tdd_fixture_runner.py @@ -48,12 +48,23 @@ def _extract_message( return message -def run_fixture(fixture_name: str) -> tuple[str, str]: - """Run a fixture ``.robot`` file and return ``(status, message)``. +def _run_fixture_impl( + fixture_name: str, + *, + extra_args: tuple[str, ...] = (), +) -> tuple[str, str]: + """Execute a fixture and return its ``(status, message)`` tuple. - Returns the status and message of the *first* test case found in - the Robot output XML. + Args: + fixture_name: Name of the fixture file without the ``.robot`` suffix. + extra_args: Additional command-line arguments passed to ``robot``. + + Returns: + A tuple containing the Robot test status and message for the first + test case in the output XML. """ + _validate_fixture_name(fixture_name) + fixture_path = _FIXTURES / f"{fixture_name}.robot" if not fixture_path.exists(): print(f"ERROR: fixture not found: {fixture_path}", file=sys.stderr) @@ -67,6 +78,7 @@ def run_fixture(fixture_name: str) -> tuple[str, str]: "robot", "--listener", _LISTENER, + *extra_args, "--outputdir", tmpdir, "--loglevel", @@ -100,7 +112,16 @@ def run_fixture(fixture_name: str) -> tuple[str, str]: print(f"stderr: {proc.stderr}", file=sys.stderr) sys.exit(1) - tree = ET.parse(out_xml) + try: + tree = ET.parse(out_xml) + except ET.ParseError as exc: + print( + f"ERROR: Failed to parse output.xml for {fixture_name!r}: {exc}", + file=sys.stderr, + ) + print(f"stderr: {proc.stderr}", file=sys.stderr) + return "ERROR", f"Malformed Robot output: {exc}" + root = tree.getroot() test_el = root.find(".//test") if test_el is None: @@ -113,6 +134,39 @@ def run_fixture(fixture_name: str) -> tuple[str, str]: return status, message +def _validate_fixture_name(fixture_name: str) -> None: + """Ensure the fixture name does not contain directory traversal tokens.""" + if ( + not fixture_name + or any(token in fixture_name for token in ("/", "\\", "..")) + or fixture_name.startswith(":") + ): + raise ValueError(f"Invalid fixture name: {fixture_name!r}") + + +def run_fixture(fixture_name: str) -> tuple[str, str]: + """Run a fixture ``.robot`` file and return ``(status, message)``. + + Returns the status and message of the *first* test case found in + the Robot output XML. + """ + return _run_fixture_impl(fixture_name) + + +def run_fixture_dryrun(fixture_name: str) -> tuple[str, str]: + """Run a fixture ``.robot`` file in dry-run mode and return ``(status, message)``. + + Dry-run mode (``--dryrun``) causes Robot Framework to validate + keyword calls without executing them. Tests get status ``NOT RUN`` + (or ``FAIL`` if keyword resolution fails). This function is used + to verify the listener's dry-run guard. + + Returns the status and message of the *first* test case found in + the Robot output XML. + """ + return _run_fixture_impl(fixture_name, extra_args=("--dryrun",)) + + def run_multi_fixture( *fixture_names: str, ) -> dict[str, tuple[str, str]]: @@ -125,6 +179,7 @@ def run_multi_fixture( """ fixture_paths = [] for name in fixture_names: + _validate_fixture_name(name) fp = _FIXTURES / f"{name}.robot" if not fp.exists(): print(f"ERROR: fixture not found: {fp}", file=sys.stderr) @@ -171,7 +226,21 @@ def run_multi_fixture( print(f"stderr: {proc.stderr}", file=sys.stderr) sys.exit(1) - tree = ET.parse(out_xml) + try: + tree = ET.parse(out_xml) + except ET.ParseError as exc: + print( + f"ERROR: Failed to parse output.xml for multi-fixture run: {exc}", + file=sys.stderr, + ) + print(f"stderr: {proc.stderr}", file=sys.stderr) + return { + "__multi_fixture_parse_error__": ( + "ERROR", + f"Malformed Robot output: {exc}", + ) + } + root = tree.getroot() results: dict[str, tuple[str, str]] = {} for test_el in root.findall(".//test"): diff --git a/robot/e2e/m5_acceptance.robot b/robot/e2e/m5_acceptance.robot index e4f8d13e1..cf4a589d7 100644 --- a/robot/e2e/m5_acceptance.robot +++ b/robot/e2e/m5_acceptance.robot @@ -108,8 +108,8 @@ Plan Test Setup # ----------------------------------------------------------------------- Context Assembly — Add Files To Context [Documentation] ``context-load`` adds files; ``context list`` reflects them. - [Tags] E2E tdd_issue tdd_issue_4189 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Tags] E2E tdd_issue tdd_issue_4189 + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete ${result}= Run CLI context-load main.py utils.py Should Be Equal As Integers ${result.rc} 0 @@ -119,8 +119,8 @@ Context Assembly — Add Files To Context Context Assembly — Show File Content [Documentation] ``context show `` displays file content. - [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Tags] E2E tdd_issue tdd_issue_4188 + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Ensure files are loaded (idempotent) Run CLI context-load main.py @@ -133,8 +133,8 @@ Context Assembly — Show Context Summary ... keywords, and that the CLI exit code is 0. Also ... checks the output is not just an error message by ... verifying the absence of common error indicators. - [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Tags] E2E tdd_issue tdd_issue_4188 + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete Run CLI context-load main.py utils.py ${result}= Run CLI context show @@ -152,8 +152,8 @@ Context Assembly — Clear Context [Documentation] ``context clear --yes`` removes all loaded files. ... Asserts files are present before clearing so the ... ``Should Not Contain`` checks are not vacuously true. - [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Tags] E2E tdd_issue tdd_issue_4188 + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete Run CLI context-load config.py # Verify precondition: config.py IS loaded before we clear @@ -187,7 +187,7 @@ Context Scaling — Structural Plumbing for 10K File Projects ... without timeout"* is deferred until the full ACMS ... indexing pipeline is wired. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Generate 10,000 tiny .py files in a subdirectory ${scale_dir}= Set Variable ${WS}${/}scale_src @@ -247,7 +247,7 @@ Context Policy - Set Default View [Documentation] Configure the default context view with include/exclude ... paths and file-size limits. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} + [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} ... msg=Prerequisite not met: policy project not created ${result}= Run CLI ... project context set ${PROJECT_POLICY} @@ -262,7 +262,7 @@ Context Policy - Set Strategize View Override [Documentation] The strategize view overrides the default with tighter ... limits and additional include paths. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} + [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} ... msg=Prerequisite not met: policy project not created ${result}= Run CLI ... project context set ${PROJECT_POLICY} @@ -278,7 +278,7 @@ Context Policy - Verify Default View ... avoid substring collisions (e.g. ``1024`` matching ... inside ``10240``). [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} + [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} ... msg=Prerequisite not met: policy project not created ${result}= Run CLI ... project context show ${PROJECT_POLICY} @@ -300,7 +300,7 @@ Context Policy - Verify Strategize View [Documentation] ``project context show --view strategize`` returns the ... overridden values. Uses parsed JSON assertions. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} + [Setup] Variable Should Exist ${POLICY_PROJECT_CREATED} ... msg=Prerequisite not met: policy project not created ${result}= Run CLI ... project context show ${PROJECT_POLICY} @@ -336,7 +336,7 @@ Budget Enforcement — Verify Constraints Stored ... Uses parsed JSON assertions to avoid substring ... collisions (e.g. ``1024`` matching inside ``10240``). [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${BUDGET_PROJECT_CREATED} + Variable Should Exist ${BUDGET_PROJECT_CREATED} ... msg=Prerequisite not met: budget project not created ${result}= Run CLI ... project context show ${PROJECT_BUDGET} @@ -364,7 +364,7 @@ Budget Enforcement — Simulate Context Assembly (Structural) ... exclusion) requires the full ACMS indexing pipeline and ... is deferred to a follow-up issue. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${BUDGET_PROJECT_CREATED} + Variable Should Exist ${BUDGET_PROJECT_CREATED} ... msg=Prerequisite not met: budget project not created ${result}= Run CLI ... project context simulate ${PROJECT_BUDGET} @@ -420,7 +420,7 @@ Context Analysis - Inspect Context Tiers (Structural) ... process. These assertions verify JSON schema correctness, ... not ACMS behavioral state. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${ANALYSIS_PROJECT_CREATED} + Variable Should Exist ${ANALYSIS_PROJECT_CREATED} ... msg=Prerequisite not met: analysis project not created ${result}= Run CLI ... project context inspect ${PROJECT_ANALYSIS} @@ -464,7 +464,7 @@ Context Analysis - Simulate Produces Structured Output ... is 0.0. These assertions verify JSON schema ... correctness, not ACMS behavioral output. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${ANALYSIS_PROJECT_CREATED} + Variable Should Exist ${ANALYSIS_PROJECT_CREATED} ... msg=Prerequisite not met: analysis project not created ${result}= Run CLI ... project context simulate ${PROJECT_ANALYSIS} @@ -491,7 +491,7 @@ Context Analysis - Show Full Policy With ACMS Config ... Uses parsed JSON assertions to avoid substring collisions ... (e.g. 500 matching inside 3500). [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Variable Should Exist ${ANALYSIS_PROJECT_CREATED} + Variable Should Exist ${ANALYSIS_PROJECT_CREATED} ... msg=Prerequisite not met: analysis project not created ${result}= Run CLI ... project context show ${PROJECT_ANALYSIS} @@ -529,7 +529,7 @@ Plan Execution — Create Project And Configure ACMS Plan Execution — Create Action [Documentation] Create an action definition from YAML config. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Plan Test Setup PLAN_PROJECT_CREATED + Plan Test Setup PLAN_PROJECT_CREATED ${action_yaml}= Catenate SEPARATOR=\n ... name: local/m5-e2e-action ... description: M5 E2E acceptance test action @@ -551,7 +551,7 @@ Plan Execution — Create Plan With Plan Use ... ``Extract JSON From Stdout`` keyword consistently with ... the rest of the suite (instead of fragile ``rindex``). [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Plan Test Setup PLAN_ACTION_CREATED + Plan Test Setup PLAN_ACTION_CREATED ${result}= Run CLI ... plan use local/m5-e2e-action ${PROJECT_PLAN} ... --format json timeout=300s @@ -573,7 +573,7 @@ Plan Execution — Resume Plan For LLM Processing ... report the actual missing field, not a misleading ... "did not return valid JSON" message. [Tags] E2E tdd_issue tdd_issue_4188 tdd_expected_fail - [Setup] Plan Test Setup PLAN_ID + Plan Test Setup PLAN_ID ${result}= Run CLI ... plan resume ${PLAN_ID} --format json ... timeout=300s diff --git a/robot/e2e/tdd_acms_behavioral_validation.robot b/robot/e2e/tdd_acms_behavioral_validation.robot index 210b673c4..0e4a7caeb 100644 --- a/robot/e2e/tdd_acms_behavioral_validation.robot +++ b/robot/e2e/tdd_acms_behavioral_validation.robot @@ -90,7 +90,7 @@ Context Simulate Returns Non-Empty Tier Data ... [Tags] tdd_bug tdd_bug_1028 tdd_issue tdd_issue_1028 E2E tdd_issue tdd_issue_4306 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Create project and configure context policy Run CLI project create ${PROJECT_SIMULATE} @@ -122,7 +122,7 @@ Context Inspect Shows Indexed Resources ... [Tags] tdd_bug tdd_bug_1028 tdd_issue tdd_issue_1028 E2E tdd_issue tdd_issue_4306 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Create project and configure context policy Run CLI project create ${PROJECT_INSPECT} @@ -155,7 +155,7 @@ Budget Enforcement Excludes Oversized Files ... [Tags] tdd_bug tdd_bug_1028 tdd_issue tdd_issue_1028 E2E tdd_issue tdd_issue_4306 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Create project with tight max_file_size (1024 bytes) Run CLI project create ${PROJECT_BUDGET} @@ -193,7 +193,7 @@ Large Project Indexes Without Timeout ... [Tags] tdd_bug tdd_bug_1028 tdd_issue tdd_issue_1028 E2E tdd_issue tdd_issue_4306 tdd_expected_fail - [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} ... msg=Prerequisite not met: suite setup did not complete # Generate 10,000 tiny .py files in a subdirectory ${scale_dir}= Set Variable ${WS}${/}scale_src diff --git a/robot/fixtures/tdd_expected_fail_infra_error.robot b/robot/fixtures/tdd_expected_fail_infra_error.robot new file mode 100644 index 000000000..f7a4aefea --- /dev/null +++ b/robot/fixtures/tdd_expected_fail_infra_error.robot @@ -0,0 +1,17 @@ +*** Settings *** +Documentation Fixture: a tdd_expected_fail test that fails with an +... infrastructure error (non-existent keyword). +... Used by helper_tdd_tag_validation.py to verify the +... non-assertion failure guard — the listener should NOT +... invert this failure because it is not a legitimate +... assertion failure from the captured bug. +... This file must NOT be picked up by the main pabot runner -- +... it is excluded via the tdd_fixture tag. + +*** Test Cases *** +Bug 995 Infrastructure Error Should Not Be Inverted + [Tags] tdd_issue tdd_issue_995 tdd_expected_fail tdd_fixture + [Documentation] This test calls a non-existent keyword, triggering + ... a "No keyword with name" error. The listener should detect this + ... as an infrastructure error and skip inversion. + Non Existent Keyword That Triggers Resolution Error diff --git a/robot/fixtures/tdd_expected_fail_setup_error.robot b/robot/fixtures/tdd_expected_fail_setup_error.robot new file mode 100644 index 000000000..fbdcb2f8e --- /dev/null +++ b/robot/fixtures/tdd_expected_fail_setup_error.robot @@ -0,0 +1,21 @@ +*** Settings *** +Documentation Fixture: a tdd_expected_fail test whose setup fails. +... Used by helper_tdd_tag_validation.py to verify the +... setup/teardown error guard — the listener should NOT +... invert this failure because the test body never executed. +... This file must NOT be picked up by the main pabot runner -- +... it is excluded via the tdd_fixture tag. + +*** Keywords *** +Failing Setup Keyword + [Documentation] Simulates an infrastructure setup failure. + Fail Setup failed: database connection unavailable + +*** Test Cases *** +Bug 996 Setup Error Should Not Be Inverted + [Tags] tdd_issue tdd_issue_996 tdd_expected_fail tdd_fixture + [Documentation] This test has a failing setup. The listener should + ... detect the setup failure and skip inversion — the FAIL result + ... should be preserved as-is. + [Setup] Failing Setup Keyword + Log This should not execute because setup failed. diff --git a/robot/fixtures/tdd_expected_fail_teardown_error.robot b/robot/fixtures/tdd_expected_fail_teardown_error.robot new file mode 100644 index 000000000..068e71b28 --- /dev/null +++ b/robot/fixtures/tdd_expected_fail_teardown_error.robot @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Fixture: a tdd_expected_fail test whose teardown fails. +... Verifies the listener's teardown guard keeps the FAIL +... status instead of inverting when cleanup breaks. +... Tagged with ``tdd_fixture`` so the main Robot suites do +... not execute it directly. + +*** Keywords *** +Failing Teardown Keyword + [Documentation] Simulates a teardown failure occurring after the + ... test body has already passed. + Fail Teardown failed: resource cleanup crashed + +*** Test Cases *** +Bug 997 Teardown Error Should Not Be Inverted + [Tags] tdd_issue tdd_issue_997 tdd_expected_fail tdd_fixture + [Documentation] This test exercises a teardown failure. The listener + ... should detect the teardown failure and skip inversion — the FAIL + ... result must remain intact. + Log Test body execution should still run before teardown. + Should Be Equal As Integers 1 1 + [Teardown] Failing Teardown Keyword diff --git a/robot/helper_tdd_guard_commands.py b/robot/helper_tdd_guard_commands.py new file mode 100644 index 000000000..1d1cc892b --- /dev/null +++ b/robot/helper_tdd_guard_commands.py @@ -0,0 +1,79 @@ +"""Guard command helpers for Robot TDD listener validation fixtures.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable + +from _tdd_fixture_runner import run_fixture, run_fixture_dryrun + +__all__ = [ + "GUARD_COMMANDS", + "cmd_dry_run_guard", + "cmd_infra_error_guard", + "cmd_setup_error_guard", + "cmd_teardown_error_guard", +] + + +def cmd_setup_error_guard() -> int: + """Verify setup error guard prevents inversion.""" + status, message = run_fixture("tdd_expected_fail_setup_error") + if status == "FAIL" and "Setup failed" in message: + print("tdd-setup-error-guard-ok") + return 0 + print( + f"FAIL: Expected FAIL with 'Setup failed' message but got " + f"{status}. Message: {message}", + file=sys.stderr, + ) + return 1 + + +def cmd_infra_error_guard() -> int: + """Verify infrastructure error guard prevents inversion.""" + status, message = run_fixture("tdd_expected_fail_infra_error") + if status == "FAIL" and "No keyword with name" in message: + print("tdd-infra-error-guard-ok") + return 0 + print( + f"FAIL: Expected FAIL with 'No keyword with name' message but " + f"got {status}. Message: {message}", + file=sys.stderr, + ) + return 1 + + +def cmd_teardown_error_guard() -> int: + """Verify teardown error guard prevents inversion.""" + status, message = run_fixture("tdd_expected_fail_teardown_error") + if status == "FAIL" and "Teardown failed" in message: + print("tdd-teardown-error-guard-ok") + return 0 + print( + f"FAIL: Expected FAIL with 'Teardown failed' message but got " + f"{status}. Message: {message}", + file=sys.stderr, + ) + return 1 + + +def cmd_dry_run_guard() -> int: + """Verify dry-run guard prevents inversion.""" + status, message = run_fixture_dryrun("tdd_expected_fail_fails") + if status == "PASS": + print("tdd-dry-run-guard-ok") + return 0 + print( + f"FAIL: Expected PASS in dry-run mode but got {status}. Message: {message}", + file=sys.stderr, + ) + return 1 + + +GUARD_COMMANDS: dict[str, Callable[[], int]] = { + "setup-error-guard": cmd_setup_error_guard, + "infra-error-guard": cmd_infra_error_guard, + "teardown-error-guard": cmd_teardown_error_guard, + "dry-run-guard": cmd_dry_run_guard, +} diff --git a/robot/helper_tdd_tag_validation.py b/robot/helper_tdd_tag_validation.py index 827e74e3d..5e7b32e60 100644 --- a/robot/helper_tdd_tag_validation.py +++ b/robot/helper_tdd_tag_validation.py @@ -13,9 +13,11 @@ from __future__ import annotations __all__: list[str] = [] +import importlib import sys from collections.abc import Callable from pathlib import Path +from typing import cast # Ensure the project root is importable so ``features.environment`` resolves. _ROOT = str(Path(__file__).resolve().parents[1]) @@ -284,43 +286,22 @@ def cmd_missing_issue_n_validation() -> int: return 1 -def cmd_normal_test_unaffected() -> int: - """Verify that a normal test (no TDD tags) is unaffected by a loaded listener. +def _check_companion_inverted( + results: dict[str, tuple[str, str]], +) -> int | None: + """Verify the companion expected-fail fixture was inverted to PASS. - Runs the normal-test fixture together with a tdd_expected_fail fixture - in a single Robot invocation. This proves the listener is loaded - (the expected-fail fixture is inverted) but does NOT modify the normal - test. + Args: + results: Mapping of test name to ``(status, message)`` tuples. + + Returns: + ``None`` if the companion fixture was inverted successfully, or ``1`` if + validation failed (after printing the error message). """ - results = run_multi_fixture( - "tdd_normal_test", - "tdd_expected_fail_fails", - ) - - # The normal test must be PASS. - normal_name = "Normal Test Unaffected By Listener" - if normal_name not in results: - print( - f"FAIL: Normal test '{normal_name}' not found in results. " - f"Available: {list(results.keys())}", - file=sys.stderr, - ) - return 1 - normal_status, normal_msg = results[normal_name] - if normal_status != "PASS": - print( - f"FAIL: Expected normal test PASS but got {normal_status}. " - f"Message: {normal_msg}", - file=sys.stderr, - ) - return 1 - - # The expected-fail fixture must have been inverted to PASS (proving - # the listener is loaded and actively processing). inverted_name = "Bug 999 Expected Failure Is Inverted To Pass" if inverted_name not in results: print( - f"FAIL: Expected-fail fixture '{inverted_name}' not found. " + f"FAIL: Companion '{inverted_name}' not found. " f"Available: {list(results.keys())}", file=sys.stderr, ) @@ -328,36 +309,44 @@ def cmd_normal_test_unaffected() -> int: inv_status, _ = results[inverted_name] if inv_status != "PASS": print( - f"FAIL: Expected-fail fixture was NOT inverted to PASS " - f"(got {inv_status}). Listener may not be loaded.", + f"FAIL: Companion not inverted to PASS (got {inv_status}). " + f"Listener may not be loaded.", file=sys.stderr, ) return 1 + return None + +def cmd_normal_test_unaffected() -> int: + """Verify normal test (no TDD tags) is unaffected by the listener.""" + results = run_multi_fixture("tdd_normal_test", "tdd_expected_fail_fails") + normal_name = "Normal Test Unaffected By Listener" + if normal_name not in results: + print(f"FAIL: '{normal_name}' not found.", file=sys.stderr) + return 1 + normal_status, normal_msg = results[normal_name] + if normal_status != "PASS": + print( + f"FAIL: Expected PASS but got {normal_status}. Message: {normal_msg}", + file=sys.stderr, + ) + return 1 + err = _check_companion_inverted(results) + if err is not None: + return err print("tdd-normal-test-unaffected-ok") return 0 def cmd_skip_status_unchanged() -> int: - """Verify that a skipped tdd_expected_fail test stays SKIP. - - Runs alongside a tdd_expected_fail fixture to prove the listener - is loaded and selectively applies — the SKIP test must stay SKIP - while the companion is correctly inverted to PASS. - """ + """Verify skipped tdd_expected_fail test stays SKIP.""" results = run_multi_fixture( "tdd_expected_fail_skip", "tdd_expected_fail_fails", ) - - # The SKIP test must remain SKIP. skip_name = "Bug 997 Skipped Expected Fail Test Stays Skip" if skip_name not in results: - print( - f"FAIL: Skip fixture '{skip_name}' not found. " - f"Available: {list(results.keys())}", - file=sys.stderr, - ) + print(f"FAIL: '{skip_name}' not found.", file=sys.stderr) return 1 skip_status, skip_msg = results[skip_name] if skip_status != "SKIP": @@ -366,38 +355,21 @@ def cmd_skip_status_unchanged() -> int: file=sys.stderr, ) return 1 - - # The expected-fail companion must be inverted to PASS (proving - # the listener is loaded and actively processing). - inverted_name = "Bug 999 Expected Failure Is Inverted To Pass" - if inverted_name not in results: - print( - f"FAIL: Companion fixture '{inverted_name}' not found. " - f"Available: {list(results.keys())}", - file=sys.stderr, - ) - return 1 - inv_status, _ = results[inverted_name] - if inv_status != "PASS": - print( - f"FAIL: Companion fixture was NOT inverted to PASS " - f"(got {inv_status}). Listener may not be loaded.", - file=sys.stderr, - ) - return 1 - + err = _check_companion_inverted(results) + if err is not None: + return err print("tdd-skip-status-unchanged-ok") return 0 def cmd_expected_fail_alone_validation() -> int: - """Verify tdd_expected_fail alone (no tdd_issue or tdd_issue_N) fails validation.""" + """Verify tdd_expected_fail alone fails validation.""" status, message = run_fixture("tdd_expected_fail_alone") if status == "FAIL" and "tdd_issue" in message and "tdd_issue_" in message: print("tdd-expected-fail-alone-validation-ok") return 0 print( - f"FAIL: Expected FAIL mentioning both tdd_issue and tdd_issue_ " + f"FAIL: Expected FAIL mentioning tdd_issue and tdd_issue_ " f"but got {status}. Message: {message}", file=sys.stderr, ) @@ -405,58 +377,32 @@ def cmd_expected_fail_alone_validation() -> int: def cmd_tdd_issue_alone_valid() -> int: - """Verify tdd_issue alone (no tdd_issue_N or tdd_expected_fail) is valid. - - Runs alongside a tdd_expected_fail fixture to prove the listener - is loaded — the tdd_issue-alone test must stay PASS while the - companion is correctly inverted. - """ - results = run_multi_fixture( - "tdd_issue_alone", - "tdd_expected_fail_fails", - ) - - # The tdd_issue-alone test must be PASS (listener should not modify). + """Verify tdd_issue alone is valid (listener does not interfere).""" + results = run_multi_fixture("tdd_issue_alone", "tdd_expected_fail_fails") alone_name = "TDD Issue Tag Alone Is Valid" if alone_name not in results: - print( - f"FAIL: tdd_issue-alone fixture '{alone_name}' not found. " - f"Available: {list(results.keys())}", - file=sys.stderr, - ) + print(f"FAIL: '{alone_name}' not found.", file=sys.stderr) return 1 alone_status, alone_msg = results[alone_name] if alone_status != "PASS": print( - f"FAIL: Expected PASS for tdd_issue-alone test but got " - f"{alone_status}. Message: {alone_msg}", + f"FAIL: Expected PASS but got {alone_status}. Message: {alone_msg}", file=sys.stderr, ) return 1 - - # The expected-fail companion must be inverted to PASS (proving - # the listener is loaded and actively processing). - inverted_name = "Bug 999 Expected Failure Is Inverted To Pass" - if inverted_name not in results: - print( - f"FAIL: Companion fixture '{inverted_name}' not found. " - f"Available: {list(results.keys())}", - file=sys.stderr, - ) - return 1 - inv_status, _ = results[inverted_name] - if inv_status != "PASS": - print( - f"FAIL: Companion fixture was NOT inverted to PASS " - f"(got {inv_status}). Listener may not be loaded.", - file=sys.stderr, - ) - return 1 - + err = _check_companion_inverted(results) + if err is not None: + return err print("tdd-issue-alone-valid-ok") return 0 +_helper_guard_module = importlib.import_module("helper_tdd_guard_commands") +GUARD_COMMANDS = cast( + dict[str, Callable[[], int]], + _helper_guard_module.GUARD_COMMANDS, +) + _COMMANDS: dict[str, Callable[[], int]] = { # Behave-side commands "validate_tags_valid_combos": validate_tags_valid_combos, @@ -486,6 +432,8 @@ _COMMANDS: dict[str, Callable[[], int]] = { "tdd-issue-alone-valid": cmd_tdd_issue_alone_valid, } +_COMMANDS.update(GUARD_COMMANDS) + def main() -> None: """Dispatch to the requested sub-command.""" diff --git a/robot/tdd_expected_fail_listener.py b/robot/tdd_expected_fail_listener.py index 144259880..8a01bf941 100644 --- a/robot/tdd_expected_fail_listener.py +++ b/robot/tdd_expected_fail_listener.py @@ -32,6 +32,33 @@ Tag Validation Rules * ``tdd_expected_fail`` requires both ``tdd_issue`` and at least one ``tdd_issue_``. +Guard Logic (parallels Behave ``apply_tdd_inversion()``) +-------------------------------------------------------- +Before inverting a ``tdd_expected_fail`` result, three guards are +checked. If any guard triggers, the original result is preserved +(inversion is skipped) because the failure is not evidence about the +captured bug: + +1. **Setup/teardown error guard** — if ``result.setup`` or + ``result.teardown`` has status ``FAIL``, the test body did not + execute (or cleanup failed). The failure is infrastructure, not + the bug. Detected via ``_has_setup_teardown_failure()``. + +2. **Non-assertion failure guard** — if ``result.message`` contains + patterns characteristic of infrastructure errors (e.g., + ``No keyword with name``, ``TypeError:``, ``ImportError:``, + ``TimeoutError:``, ``Connection refused``), the failure is not a + legitimate assertion failure from the captured bug. Detected via + ``_is_infrastructure_error()``. + +3. **Dry-run guard** — if all body keywords have status ``NOT RUN`` + (Robot Framework dry-run mode), no test actually executed, so the + result is meaningless. Inversion is skipped. + +These guards mirror the Behave ``apply_tdd_inversion()`` guards in +``features/environment.py`` (hook-error guard, dry-run guard, +non-assertion exception guard). + Implementation Notes -------------------- The listener uses the Robot Framework Listener v3 API with module-level @@ -148,6 +175,105 @@ def _should_invert_result(tags: set[str]) -> bool: return "tdd_expected_fail" in tags +# --------------------------------------------------------------------------- +# Infrastructure error patterns — used by ``_is_infrastructure_error()`` +# to detect non-assertion failures that should NOT be inverted. +# +# These patterns are checked against ``result.message`` (case-insensitive). +# Only clear infrastructure indicators are listed; unknown failure patterns +# are assumed to be assertion failures and ARE inverted (conservative +# approach matching the Behave ``apply_tdd_inversion()`` philosophy). +# --------------------------------------------------------------------------- +_INFRA_ERROR_PATTERNS: tuple[str, ...] = ( + # Robot keyword resolution errors + "no keyword with name", + # Python exception types from libraries + "typeerror:", + "importerror:", + "modulenotfounderror:", + "filenotfounderror:", + "timeouterror:", + "permissionerror:", + "oserror:", + "connectionerror:", + "attributeerror:", + # Network / connectivity errors + # WARNING: these patterns can false-positive if a test asserts on + # specific network error substrings in its expected output. + "connection refused", + "connection reset", + "connection timed out", + # Robot-specific infrastructure messages + "no library with name", + "importing library", + "variable '${", # RF scalar variable resolution error + "variable '@{", # RF list variable resolution error + "variable '&{", # RF dict variable resolution error + "variable '%{", # RF environment variable resolution error +) + + +def _has_setup_teardown_failure(result: ResultTestCase) -> bool: + """Return ``True`` if the test failed due to setup or teardown. + + In Robot Framework, when a test setup fails the test body does not + execute — the failure is infrastructure, not the captured bug. + Similarly, a teardown failure indicates cleanup problems, not bug + evidence. Note that Robot can report a teardown failure even when the + body passed; the guard still returns ``True`` to avoid inverting a + result that represents cleanup breakage rather than a fixed bug. + + Checks two indicators: + + 1. ``result.setup.status`` or ``result.teardown.status`` is + ``"FAIL"`` (structural check via the RF result model). + 2. ``result.message`` starts with ``"Setup failed:"`` or + ``"Teardown failed:"`` (string check as a fallback for RF + versions that may not populate the structural attributes). + + Args: + result: The Robot Framework test result object. + + Returns: + ``True`` if a setup or teardown failure was detected. + """ + # Structural check: RF result model exposes setup/teardown objects. + if result.setup and result.setup.status == "FAIL": + return True + if result.teardown and result.teardown.status == "FAIL": + return True + + # String-based fallback: RF prefixes the message when setup/teardown + # fails. + msg = result.message or "" + return msg.startswith("Setup failed:") or msg.startswith("Teardown failed:") + + +def _is_infrastructure_error(message: str) -> bool: + """Return ``True`` if the failure message indicates an infrastructure error. + + Infrastructure errors (import failures, keyword resolution errors, + timeouts, connection problems, etc.) are NOT evidence about the + captured bug and should not be inverted by the TDD listener. + + The check is conservative: only well-known infrastructure patterns + trigger a match. Unknown failure messages are assumed to be + legitimate assertion failures and will be inverted normally. + + Args: + message: The ``result.message`` from the Robot test result. + + Returns: + ``True`` if the message matches a known infrastructure error + pattern. + """ + if not message: + return False + + lower_msg = message.lower() + return any(pattern in lower_msg for pattern in _INFRA_ERROR_PATTERNS) + + def start_test( data: RunningTestCase, result: ResultTestCase, @@ -175,7 +301,19 @@ def end_test( forced to fail with the validation error message regardless of the actual test outcome. - For ``tdd_expected_fail`` tests with valid tags: + For ``tdd_expected_fail`` tests with valid tags, three guards are + checked before inversion (paralleling the Behave + ``apply_tdd_inversion()`` guards): + + 1. **Setup/teardown error guard** — if the test failed due to a + setup or teardown error, inversion is skipped. + 2. **Non-assertion failure guard** — if ``result.message`` matches + known infrastructure error patterns, inversion is skipped. + 3. **Dry-run guard** — if all body keywords have status ``NOT RUN`` + (dry-run mode), inversion is skipped. + + If no guard triggers: + * If the test **failed** (bug still exists), the result is inverted to **PASS** (expected failure). * If the test **passed** (bug appears fixed), the result is inverted @@ -214,6 +352,51 @@ def end_test( if not _should_invert_result(tags): return + # --- Guard 1: Setup/teardown error --- + # Never invert when the failure is from test setup or teardown. + # The test body did not execute (or cleanup failed), so the result + # is not evidence about the captured bug. + if _has_setup_teardown_failure(result): + message_snippet = (result.message or "")[:200] + _logger.warning( + "TDD expected-fail test '%s' has a setup/teardown failure " + "— not inverting. Message: %s", + full_name, + message_snippet, + ) + return + + # --- Guard 2: Non-assertion failure detection --- + # If the test failed with a message matching known infrastructure + # error patterns (import errors, keyword resolution failures, + # timeouts, etc.), do not invert — the failure is not the captured + # bug. + if result.status == "FAIL" and _is_infrastructure_error(result.message): + message_snippet = (result.message or "")[:200] + _logger.warning( + "TDD expected-fail test '%s' failed with an infrastructure " + "error — not inverting. Message: %s", + full_name, + message_snippet, + ) + return + + # --- Guard 3: Dry-run mode --- + # In dry-run mode, Robot Framework validates keyword existence but + # does not execute them. The test status is PASS (no failure + # occurred) but the body keywords have status NOT RUN. Detect + # dry-run by checking if ALL body keywords are NOT RUN — if so, + # no test actually executed and the result is meaningless. + if result.body and all( + getattr(item, "status", None) == "NOT RUN" for item in result.body + ): + _logger.debug( + "TDD expected-fail test '%s' appears to be in dry-run mode " + "(all body keywords NOT RUN) — leaving unchanged.", + full_name, + ) + return + if result.status == "FAIL": # Expected failure -- the bug still exists. Invert to PASS. _logger.info( diff --git a/robot/tdd_tag_validation.robot b/robot/tdd_tag_validation.robot index 482f1acf5..ed782779f 100644 --- a/robot/tdd_tag_validation.robot +++ b/robot/tdd_tag_validation.robot @@ -230,3 +230,52 @@ TDD Issue Tag Alone Is Valid Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} tdd-issue-alone-valid-ok + +# =========================================================================== +# Robot-side guard tests (tdd_expected_fail_listener.py guards) +# =========================================================================== + +TDD Expected Fail Setup Error Guard Prevents Inversion + [Documentation] A ``tdd_expected_fail`` test whose setup fails should NOT + ... have its result inverted. The setup/teardown error guard should detect + ... the setup failure and preserve the original FAIL status. + [Tags] tdd_infrastructure guard + ${result}= Run Process ${PYTHON} ${HELPER} setup-error-guard cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-setup-error-guard-ok + +TDD Expected Fail Teardown Error Guard Prevents Inversion + [Documentation] A ``tdd_expected_fail`` test whose teardown fails should NOT + ... be inverted. Even though the body may have passed, the teardown failure + ... indicates cleanup issues, so the guard must preserve the FAIL status. + [Tags] tdd_infrastructure guard + ${result}= Run Process ${PYTHON} ${HELPER} teardown-error-guard cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-teardown-error-guard-ok + +TDD Expected Fail Infrastructure Error Guard Prevents Inversion + [Documentation] A ``tdd_expected_fail`` test that fails with a non-assertion + ... infrastructure error (e.g., missing keyword) should NOT have its result + ... inverted. The non-assertion failure guard should detect the infrastructure + ... error pattern and preserve the original FAIL status. + [Tags] tdd_infrastructure guard + ${result}= Run Process ${PYTHON} ${HELPER} infra-error-guard cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-infra-error-guard-ok + +TDD Expected Fail Dry Run Guard Prevents Inversion + [Documentation] A ``tdd_expected_fail`` test run in dry-run mode should NOT + ... have its result inverted. The dry-run guard should observe the PASS + ... status emitted during keyword validation and leave it untouched. + [Tags] tdd_infrastructure guard + ${result}= Run Process ${PYTHON} ${HELPER} dry-run-guard cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} tdd-dry-run-guard-ok