diff --git a/features/ci_workflow_validation.feature b/features/ci_workflow_validation.feature index 31da4cc47..1186854df 100644 --- a/features/ci_workflow_validation.feature +++ b/features/ci_workflow_validation.feature @@ -156,7 +156,7 @@ Feature: CI workflow validation == "failure" || == "error" checks so skipped jobs do not appear as failures. Given the CI workflow file at ".forgejo/workflows/ci.yml" When I parse the CI workflow YAML - Then any cancelled or skipped job must not cause status-check failure + Then the workflow uses == "failure" || == "error" logic so skipped jobs do not fail Scenario: Status-check passes on cancelled dependent jobs Description: >- @@ -165,7 +165,7 @@ Feature: CI workflow validation only == "failure" || == "error" results trigger a pipeline failure. Given the CI workflow file at ".forgejo/workflows/ci.yml" When I parse the CI workflow YAML - Then the job "status-check" should have == "failure" comparisons + Then the workflow uses != "success" exclusion so cancelled jobs are not treated as failures Scenario: Status-check fails on actually failed jobs Description: >- @@ -182,7 +182,7 @@ Feature: CI workflow validation This scenario verifies that the dependency list is complete. Given the CI workflow file at ".forgejo/workflows/ci.yml" When I parse the CI workflow YAML - Then the job "status-check" uses failure-based checking + And any cancelled or skipped job must not cause status-check failure # --- Coverage threshold --- # Note: #4227 is closed, but this specific scenario still fails because diff --git a/features/steps/ci_workflow_validation_steps.py b/features/steps/ci_workflow_validation_steps.py index b36a6e980..f86d95b89 100644 --- a/features/steps/ci_workflow_validation_steps.py +++ b/features/steps/ci_workflow_validation_steps.py @@ -208,7 +208,8 @@ def step_then_status_check_failure_logic(context): # Must use failure-based checks (error state also treated as failing) assert '= "failure"' in all_run_commands, ( - "status-check should use \"failure\" comparisons" + "status-check must use \"failure\" comparison checks; " + "found no failure-based logic which means skipped/cancelled jobs may cause false failures" ) assert '= "error"' in all_run_commands, ( "status-check should also use \"error\" comparisons for infrastructure failures" @@ -279,7 +280,8 @@ def step_then_actual_failures_still_detected(context, job_name): ) assert '= "failure"' in all_run_commands, ( - f"Job '{job_name}' must use \"failure\" comparisons to detect actual failures" + f"Job '{job_name}' does not use failure-based detection; " + "genuine job failures won't be properly caught by status-check" ) @@ -298,8 +300,8 @@ def step_then_failure_based_checking(context, job_name): ) assert '= "failure"' in all_run_commands, ( - f"Job '{job_name}' does NOT use failure-based checking. Skipped/cancelled/err" - "or jobs may cause false pipeline failures." + f"Job '{job_name}' does not use failure-based checking; " + "skipped/cancelled/error jobs may cause false pipeline failures" ) assert '!= "success"' not in all_run_commands, ( f"Job '{job_name}' still uses != \"success\" checks which treats "