Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3357c2c838 | |||
| 9e0435d2a4 | |||
| 8a8fd263c8 |
@@ -140,6 +140,34 @@ jobs:
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
|
||||
tdd-consistency:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for history)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Run TDD tag consistency check via nox
|
||||
run: |
|
||||
nox -s tdd_consistency
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
FORGEJO_URL: https://git.cleverthis.com
|
||||
# FORGEJO_TOKEN is used for Check 2 (Forgejo API issue-status
|
||||
# queries). Configure via repository Actions secrets.
|
||||
# If not set, Check 2 degrades gracefully to warnings.
|
||||
FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
|
||||
|
||||
unit_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Added `nox -s tdd_consistency` CI quality gate that enforces TDD bug tag
|
||||
consistency. Check 1 verifies commits closing `#N` remove
|
||||
`@tdd_expected_fail` from `@tdd_bug_N` tests. Check 2 cross-references
|
||||
tag state with Forgejo issue status (open/closed). (#966)
|
||||
|
||||
- Added TDD bug-capture E2E tests for bug #1028 — ACMS indexing pipeline not
|
||||
wired into CLI. Four Robot Framework E2E tests prove ContextTierService starts
|
||||
empty on every CLI invocation. Tests use ``@tdd_expected_fail`` until the bug
|
||||
|
||||
@@ -1222,6 +1222,18 @@ These rules are enforced by the test environment hooks and CI quality gates:
|
||||
- A bug fix PR that closes issue `#N` where no `@tdd_bug_N` test exists in the codebase is
|
||||
blocked by the CI quality gate — the TDD step was skipped.
|
||||
|
||||
#### Automated Enforcement
|
||||
|
||||
These rules are enforced by `nox -s tdd_consistency`, which runs as part of the CI pipeline:
|
||||
|
||||
- **Check 1 (commit-based)**: Scans commit messages for `#N` references and verifies that
|
||||
tests tagged `@tdd_bug_N` have had `@tdd_expected_fail` removed.
|
||||
- **Check 2 (Forgejo API)**: Cross-references tag state with Forgejo issue status — detects
|
||||
`@tdd_expected_fail` on closed issues and missing `@tdd_expected_fail` on open issues.
|
||||
|
||||
Run locally via `nox -s tdd_consistency`. Check 2 requires `FORGEJO_URL` and `FORGEJO_TOKEN`
|
||||
environment variables; it degrades to a warning when the API is unreachable.
|
||||
|
||||
### Testing Tools
|
||||
|
||||
Run tests using `nox`. Do not invoke `behave`, `robot`, or similar runners directly. If a `nox`
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Step definitions for TDD tag consistency check scenarios."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
_SCRIPTS = str(Path(__file__).resolve().parents[2] / "scripts")
|
||||
if _SCRIPTS not in sys.path:
|
||||
sys.path.insert(0, _SCRIPTS)
|
||||
|
||||
_mod = importlib.import_module("check-tdd-tag-consistency")
|
||||
TagInfo = _mod.TagInfo
|
||||
CheckResult = _mod.CheckResult
|
||||
check_commit_consistency = _mod.check_commit_consistency
|
||||
check_forgejo_consistency = _mod.check_forgejo_consistency
|
||||
get_branch_issue_references = _mod.get_branch_issue_references
|
||||
_scan_behave_file = _mod._scan_behave_file
|
||||
_scan_robot_file = _mod._scan_robot_file
|
||||
_SYNTHETIC_ISSUE_NUMBERS = _mod._SYNTHETIC_ISSUE_NUMBERS
|
||||
_ISSUE_NOT_FOUND = _mod._ISSUE_NOT_FOUND
|
||||
|
||||
|
||||
def _make_tag_info(issue_number: int, has_expected_fail: bool) -> Any:
|
||||
return TagInfo(
|
||||
file=Path("features/test_dummy.feature"),
|
||||
line=1,
|
||||
issue_number=issue_number,
|
||||
has_expected_fail=has_expected_fail,
|
||||
suite="behave-unit",
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_dir(path: str) -> None:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
|
||||
@given("tdd consistency a tag map with issue {n:d} and expected_fail is {ef}")
|
||||
def step_tag_map(context: Context, n: int, ef: str) -> None:
|
||||
if not hasattr(context, "tdd_tag_infos"):
|
||||
context.tdd_tag_infos = []
|
||||
context.tdd_tag_infos.append(_make_tag_info(n, ef.strip().lower() == "true"))
|
||||
|
||||
|
||||
@given('tdd consistency a temporary feature file with "{tags}"')
|
||||
def step_temp_feature(context: Context, tags: str) -> None:
|
||||
content = f"""{tags}\nFeature: Dummy TDD feature\n Scenario: Dummy scenario\n Given nothing\n"""
|
||||
td = tempfile.mkdtemp(prefix="tdd_consistency_test_")
|
||||
(Path(td) / "dummy.feature").write_text(content, encoding="utf-8")
|
||||
context.tdd_temp_feature = Path(td) / "dummy.feature"
|
||||
context.add_cleanup(lambda: _cleanup_dir(td))
|
||||
|
||||
|
||||
@given('tdd consistency a temporary feature file with feature-level "{tags}"')
|
||||
def step_temp_feature_level(context: Context, tags: str) -> None:
|
||||
content = f"""{tags}\nFeature: Dummy TDD feature\n\n Scenario: Inheriting tags\n Given nothing\n"""
|
||||
td = tempfile.mkdtemp(prefix="tdd_consistency_test_")
|
||||
(Path(td) / "dummy.feature").write_text(content, encoding="utf-8")
|
||||
context.tdd_temp_feature = Path(td) / "dummy.feature"
|
||||
context.add_cleanup(lambda: _cleanup_dir(td))
|
||||
|
||||
|
||||
@given("tdd consistency a temporary feature file with content")
|
||||
def step_temp_feature_multiline(context: Context) -> None:
|
||||
td = tempfile.mkdtemp(prefix="tdd_consistency_test_")
|
||||
(Path(td) / "dummy.feature").write_text(context.text or "", encoding="utf-8")
|
||||
context.tdd_temp_feature = Path(td) / "dummy.feature"
|
||||
context.add_cleanup(lambda: _cleanup_dir(td))
|
||||
|
||||
|
||||
@given("tdd consistency a temporary robot file with content")
|
||||
def step_temp_robot_multiline(context: Context) -> None:
|
||||
td = tempfile.mkdtemp(prefix="tdd_consistency_test_")
|
||||
(Path(td) / "dummy.robot").write_text(context.text or "", encoding="utf-8")
|
||||
context.tdd_temp_robot = Path(td) / "dummy.robot"
|
||||
context.add_cleanup(lambda: _cleanup_dir(td))
|
||||
|
||||
|
||||
@given("tdd consistency a mocked git log returning {text}")
|
||||
def step_mock_git_log(context: Context, text: str) -> None:
|
||||
context.tdd_mock_git_log_text = text.strip('"').strip("'")
|
||||
|
||||
|
||||
@when('tdd consistency check 1 runs with referenced issues "{refs}"')
|
||||
def step_check1(context: Context, refs: str) -> None:
|
||||
ref_set = {int(r.strip()) for r in refs.split(",")} if refs.strip() else set()
|
||||
tag_infos: list[Any] = getattr(context, "tdd_tag_infos", [])
|
||||
with patch.object(
|
||||
_mod, "get_branch_issue_references", return_value=(ref_set, None)
|
||||
):
|
||||
context.tdd_check1_result = check_commit_consistency("origin/master", tag_infos)
|
||||
|
||||
|
||||
@when("tdd consistency check 1 runs with no referenced issues")
|
||||
def step_check1_empty(context: Context) -> None:
|
||||
tag_infos: list[Any] = getattr(context, "tdd_tag_infos", [])
|
||||
with patch.object(_mod, "get_branch_issue_references", return_value=(set(), None)):
|
||||
context.tdd_check1_result = check_commit_consistency("origin/master", tag_infos)
|
||||
|
||||
|
||||
@when("tdd consistency get_branch_issue_references runs with mocked git log")
|
||||
def step_get_branch_refs(context: Context) -> None:
|
||||
mock_result = MagicMock()
|
||||
mock_result.stdout = getattr(context, "tdd_mock_git_log_text", "")
|
||||
with patch.object(_mod.subprocess, "run", return_value=mock_result):
|
||||
refs, _warning = get_branch_issue_references("origin/master")
|
||||
context.tdd_issue_refs = refs
|
||||
|
||||
|
||||
@when('tdd consistency check 2 runs with issue {n:d} state "{state}"')
|
||||
def step_check2(context: Context, n: int, state: str) -> None:
|
||||
tag_infos: list[Any] = getattr(context, "tdd_tag_infos", [])
|
||||
|
||||
def mock_fetch(url: str, issue_number: int) -> str | None:
|
||||
if issue_number in _SYNTHETIC_ISSUE_NUMBERS:
|
||||
return _ISSUE_NOT_FOUND
|
||||
return state if issue_number == n else _ISSUE_NOT_FOUND
|
||||
|
||||
with patch.object(_mod, "_fetch_issue_state", side_effect=mock_fetch):
|
||||
context.tdd_check2_result = check_forgejo_consistency(
|
||||
"https://fake.example.com", tag_infos
|
||||
)
|
||||
|
||||
|
||||
@when("tdd consistency check 2 runs with API unreachable")
|
||||
def step_check2_unreachable(context: Context) -> None:
|
||||
tag_infos: list[Any] = getattr(context, "tdd_tag_infos", [])
|
||||
with patch.object(_mod, "_fetch_issue_state", return_value=None):
|
||||
context.tdd_check2_result = check_forgejo_consistency(
|
||||
"https://unreachable.example.com", tag_infos
|
||||
)
|
||||
|
||||
|
||||
@when("tdd consistency the scanner runs on the temporary file")
|
||||
def step_scan(context: Context) -> None:
|
||||
results: list[Any] = []
|
||||
_scan_behave_file(context.tdd_temp_feature, "behave-unit", results)
|
||||
context.tdd_scan_results = results
|
||||
|
||||
|
||||
@when("tdd consistency the robot scanner runs on the temporary file")
|
||||
def step_scan_robot(context: Context) -> None:
|
||||
results: list[Any] = []
|
||||
_scan_robot_file(context.tdd_temp_robot, "robot-integration", results)
|
||||
context.tdd_scan_results = results
|
||||
|
||||
|
||||
@then("tdd consistency check 1 should have {n} violations")
|
||||
@then("tdd consistency check 1 should have {n} violation")
|
||||
def step_check1_violations(context: Context, n: str) -> None:
|
||||
actual = len(context.tdd_check1_result.violations)
|
||||
assert actual == int(n), f"Expected {n} Check 1 violation(s), got {actual}"
|
||||
|
||||
|
||||
@then("tdd consistency check 2 should have {n} violations")
|
||||
@then("tdd consistency check 2 should have {n} violation")
|
||||
def step_check2_violations(context: Context, n: str) -> None:
|
||||
actual = len(context.tdd_check2_result.violations)
|
||||
assert actual == int(n), f"Expected {n} Check 2 violation(s), got {actual}"
|
||||
|
||||
|
||||
@then('tdd consistency the violation message should contain "{text}"')
|
||||
def step_violation_contains(context: Context, text: str) -> None:
|
||||
result = getattr(context, "tdd_check2_result", None) or getattr(
|
||||
context, "tdd_check1_result", None
|
||||
)
|
||||
assert result and result.violations, "No violations to check"
|
||||
messages = " ".join(v.message for v in result.violations)
|
||||
assert text in messages, f"Expected '{text}' in: {messages}"
|
||||
|
||||
|
||||
@then("tdd consistency check 2 should have warnings")
|
||||
def step_check2_warnings(context: Context) -> None:
|
||||
assert context.tdd_check2_result.warnings, "Expected warnings but got none"
|
||||
|
||||
|
||||
@then("tdd consistency the scanner should find issue {n:d} with expected_fail {ef}")
|
||||
def step_scan_result(context: Context, n: int, ef: str) -> None:
|
||||
expected_ef = ef.strip().lower() == "true"
|
||||
matching = [r for r in context.tdd_scan_results if r.issue_number == n]
|
||||
assert matching, f"Issue #{n} not found"
|
||||
for m in matching:
|
||||
assert m.has_expected_fail == expected_ef
|
||||
|
||||
|
||||
@then("tdd consistency the scanner should find {n:d} tag entries")
|
||||
def step_scan_count(context: Context, n: int) -> None:
|
||||
assert len(context.tdd_scan_results) == n, f"Expected {n} entries"
|
||||
|
||||
|
||||
@then("tdd consistency the issue references should contain {n:d}")
|
||||
def step_issue_refs_contain(context: Context, n: int) -> None:
|
||||
assert n in context.tdd_issue_refs, f"Expected #{n}"
|
||||
|
||||
|
||||
@then("tdd consistency the issue references should be empty")
|
||||
def step_issue_refs_empty(context: Context) -> None:
|
||||
assert len(context.tdd_issue_refs) == 0, (
|
||||
f"Expected empty, got {context.tdd_issue_refs}"
|
||||
)
|
||||
@@ -0,0 +1,173 @@
|
||||
@mock_only
|
||||
Feature: TDD tag consistency checks
|
||||
Verify that the TDD tag consistency checker correctly detects
|
||||
violations between @tdd_bug_<N> / @tdd_expected_fail tags and
|
||||
issue lifecycle (commit references, Forgejo issue status).
|
||||
|
||||
# --- Check 1: Commit-based consistency ---
|
||||
|
||||
Scenario: Check 1 passes when commit references issue and tdd_expected_fail is removed
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is false
|
||||
When tdd consistency check 1 runs with referenced issues "100"
|
||||
Then tdd consistency check 1 should have 0 violations
|
||||
|
||||
Scenario: Check 1 fails when commit references issue but tdd_expected_fail is present
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 1 runs with referenced issues "100"
|
||||
Then tdd consistency check 1 should have 1 violation
|
||||
And tdd consistency the violation message should contain "still present"
|
||||
|
||||
Scenario: Check 1 passes when commit references unrelated issue
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 1 runs with referenced issues "200"
|
||||
Then tdd consistency check 1 should have 0 violations
|
||||
|
||||
Scenario: Check 1 passes when no commits reference any issue
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 1 runs with no referenced issues
|
||||
Then tdd consistency check 1 should have 0 violations
|
||||
|
||||
Scenario: Check 1 handles multiple issues and multiple tags
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
And tdd consistency a tag map with issue 200 and expected_fail is false
|
||||
And tdd consistency a tag map with issue 300 and expected_fail is true
|
||||
When tdd consistency check 1 runs with referenced issues "100,300"
|
||||
Then tdd consistency check 1 should have 2 violations
|
||||
|
||||
# --- git log / regex code path ---
|
||||
|
||||
Scenario: get_branch_issue_references extracts issue numbers from commit text
|
||||
Given tdd consistency a mocked git log returning "fix: resolve #42 and close #99"
|
||||
When tdd consistency get_branch_issue_references runs with mocked git log
|
||||
Then tdd consistency the issue references should contain 42
|
||||
And tdd consistency the issue references should contain 99
|
||||
|
||||
Scenario: get_branch_issue_references returns empty set for no references
|
||||
Given tdd consistency a mocked git log returning "chore: update dependencies"
|
||||
When tdd consistency get_branch_issue_references runs with mocked git log
|
||||
Then tdd consistency the issue references should be empty
|
||||
|
||||
Scenario: get_branch_issue_references handles hash in middle of word
|
||||
Given tdd consistency a mocked git log returning "bump version to v2#3 and fix #77"
|
||||
When tdd consistency get_branch_issue_references runs with mocked git log
|
||||
Then tdd consistency the issue references should contain 77
|
||||
|
||||
# --- Check 2: Forgejo API consistency ---
|
||||
|
||||
Scenario: Check 2 passes when expected_fail present and issue is open
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 2 runs with issue 100 state "open"
|
||||
Then tdd consistency check 2 should have 0 violations
|
||||
|
||||
Scenario: Check 2 fails when expected_fail present and issue is closed
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 2 runs with issue 100 state "closed"
|
||||
Then tdd consistency check 2 should have 1 violation
|
||||
And tdd consistency the violation message should contain "CLOSED"
|
||||
|
||||
Scenario: Check 2 passes when expected_fail absent and issue is closed
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is false
|
||||
When tdd consistency check 2 runs with issue 100 state "closed"
|
||||
Then tdd consistency check 2 should have 0 violations
|
||||
|
||||
Scenario: Check 2 fails when expected_fail absent and issue is open
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is false
|
||||
When tdd consistency check 2 runs with issue 100 state "open"
|
||||
Then tdd consistency check 2 should have 1 violation
|
||||
And tdd consistency the violation message should contain "OPEN"
|
||||
|
||||
Scenario: Check 2 degrades gracefully when API is unreachable
|
||||
Given tdd consistency a tag map with issue 100 and expected_fail is true
|
||||
When tdd consistency check 2 runs with API unreachable
|
||||
Then tdd consistency check 2 should have 0 violations
|
||||
And tdd consistency check 2 should have warnings
|
||||
|
||||
Scenario: Check 2 skips synthetic issue numbers (997-999)
|
||||
Given tdd consistency a tag map with issue 999 and expected_fail is true
|
||||
When tdd consistency check 2 runs with issue 999 state "closed"
|
||||
Then tdd consistency check 2 should have 0 violations
|
||||
|
||||
# --- Behave tag scanning ---
|
||||
|
||||
Scenario: Scanner finds tdd_bug_N tags in Behave feature files
|
||||
Given tdd consistency a temporary feature file with "@tdd_bug @tdd_bug_42 @tdd_expected_fail"
|
||||
When tdd consistency the scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 42 with expected_fail true
|
||||
|
||||
Scenario: Scanner finds tdd_bug_N tags without tdd_expected_fail
|
||||
Given tdd consistency a temporary feature file with "@tdd_bug @tdd_bug_42"
|
||||
When tdd consistency the scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 42 with expected_fail false
|
||||
|
||||
Scenario: Scanner inherits feature-level tags to scenarios
|
||||
Given tdd consistency a temporary feature file with feature-level "@tdd_bug @tdd_bug_55 @tdd_expected_fail"
|
||||
When tdd consistency the scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 55 with expected_fail true
|
||||
|
||||
Scenario: Scanner recognizes Scenario Template alias
|
||||
Given tdd consistency a temporary feature file with content
|
||||
"""
|
||||
@tdd_bug @tdd_bug_77 @tdd_expected_fail
|
||||
Feature: Template alias
|
||||
|
||||
Scenario Template: Uses alias
|
||||
Given nothing
|
||||
"""
|
||||
When tdd consistency the scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 77 with expected_fail true
|
||||
|
||||
# --- Robot file scanner ---
|
||||
|
||||
Scenario: Robot scanner parses Tags line for tdd_bug_N
|
||||
Given tdd consistency a temporary robot file with content
|
||||
"""
|
||||
*** Settings ***
|
||||
Documentation Test file
|
||||
*** Test Cases ***
|
||||
Example Test
|
||||
[Tags] tdd_bug tdd_bug_123 tdd_expected_fail
|
||||
Log hello
|
||||
"""
|
||||
When tdd consistency the robot scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 123 with expected_fail true
|
||||
And tdd consistency the scanner should find 1 tag entries
|
||||
|
||||
Scenario: Robot scanner inherits Force Tags to test cases
|
||||
Given tdd consistency a temporary robot file with content
|
||||
"""
|
||||
*** Settings ***
|
||||
Force Tags tdd_bug tdd_bug_456 tdd_expected_fail
|
||||
*** Test Cases ***
|
||||
Example Test
|
||||
[Tags] smoke
|
||||
Log hello
|
||||
"""
|
||||
When tdd consistency the robot scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 456 with expected_fail true
|
||||
|
||||
Scenario: Robot scanner handles continuation lines
|
||||
Given tdd consistency a temporary robot file with content
|
||||
"""
|
||||
*** Settings ***
|
||||
Documentation Test file
|
||||
*** Test Cases ***
|
||||
Example Test
|
||||
[Tags] tdd_bug tdd_bug_789
|
||||
... tdd_expected_fail
|
||||
Log hello
|
||||
"""
|
||||
When tdd consistency the robot scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 789 with expected_fail true
|
||||
|
||||
Scenario: Robot scanner emits Force Tags when no Tags line exists
|
||||
Given tdd consistency a temporary robot file with content
|
||||
"""
|
||||
*** Settings ***
|
||||
Force Tags tdd_bug tdd_bug_321 tdd_expected_fail
|
||||
*** Test Cases ***
|
||||
Example Test
|
||||
Log hello
|
||||
"""
|
||||
When tdd consistency the robot scanner runs on the temporary file
|
||||
Then tdd consistency the scanner should find issue 321 with expected_fail true
|
||||
And tdd consistency the scanner should find 1 tag entries
|
||||
+14
@@ -1004,6 +1004,20 @@ def dead_code(session: nox.Session):
|
||||
)
|
||||
|
||||
|
||||
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
||||
def tdd_consistency(session: nox.Session):
|
||||
"""Check TDD tag consistency across test suites."""
|
||||
session.install("-e", ".[dev]")
|
||||
forgejo_url = os.environ.get("FORGEJO_URL", "https://git.cleverthis.com")
|
||||
base_branch = os.environ.get("TDD_BASE_BRANCH", "origin/master")
|
||||
session.env["FORGEJO_URL"] = forgejo_url
|
||||
session.env["TDD_BASE_BRANCH"] = base_branch
|
||||
forgejo_token = os.environ.get("FORGEJO_TOKEN", "")
|
||||
if forgejo_token:
|
||||
session.env["FORGEJO_TOKEN"] = forgejo_token
|
||||
session.run("python", "scripts/check-tdd-tag-consistency.py")
|
||||
|
||||
|
||||
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
||||
def complexity(session: nox.Session):
|
||||
"""Check code complexity using radon."""
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Robot helper for tdd_consistency_check.robot integration tests.
|
||||
|
||||
Runs ``scripts/check-tdd-tag-consistency.py`` as a subprocess and
|
||||
verifies exit codes and output format.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
_SCRIPT = str(_PROJECT_ROOT / "scripts" / "check-tdd-tag-consistency.py")
|
||||
_SUBPROCESS_TIMEOUT = 60
|
||||
|
||||
|
||||
def _run_script(*extra_args: str) -> subprocess.CompletedProcess[str]:
|
||||
"""Run the consistency check script and return the result."""
|
||||
cmd = [
|
||||
sys.executable,
|
||||
_SCRIPT,
|
||||
"--base-branch",
|
||||
"origin/master",
|
||||
*extra_args,
|
||||
]
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_SUBPROCESS_TIMEOUT,
|
||||
cwd=str(_PROJECT_ROOT),
|
||||
)
|
||||
|
||||
|
||||
def cmd_passes_on_master() -> int:
|
||||
"""Verify the script exits 0 when run on master (no violations)."""
|
||||
proc = _run_script()
|
||||
if proc.returncode != 0:
|
||||
print(
|
||||
f"FAIL: Script exited with rc={proc.returncode}\n"
|
||||
f"stdout: {proc.stdout[:500]}\n"
|
||||
f"stderr: {proc.stderr[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if "All TDD tag consistency checks passed" not in proc.stdout:
|
||||
print(
|
||||
f"FAIL: Expected success message in output.\nstdout: {proc.stdout[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("tdd-consistency-passes-on-master-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_reports_tag_count() -> int:
|
||||
"""Verify the script reports the tag scan summary."""
|
||||
proc = _run_script()
|
||||
if "@tdd_bug_<N> tags" not in proc.stdout:
|
||||
print(
|
||||
f"FAIL: Expected tag count report in output.\nstdout: {proc.stdout[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("tdd-consistency-reports-tag-count-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_graceful_no_forgejo() -> int:
|
||||
"""Verify Check 2 degrades gracefully without Forgejo URL."""
|
||||
proc = _run_script("--forgejo-url", "")
|
||||
if proc.returncode != 0:
|
||||
print(
|
||||
f"FAIL: Script should pass when Forgejo URL is empty, "
|
||||
f"got rc={proc.returncode}\n"
|
||||
f"stderr: {proc.stderr[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if "Skipping Forgejo API check" not in proc.stdout:
|
||||
print(
|
||||
f"FAIL: Expected skip message for empty Forgejo URL.\n"
|
||||
f"stdout: {proc.stdout[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("tdd-consistency-graceful-no-forgejo-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_violation_exits_nonzero() -> int:
|
||||
"""Verify the script exits 1 when violations exist.
|
||||
|
||||
Creates a temporary .feature file with @tdd_expected_fail +
|
||||
@tdd_bug_966 and runs the script with a fake commit referencing
|
||||
#966. The script should detect the violation and exit 1.
|
||||
"""
|
||||
# Create a temp feature file with a known tdd_expected_fail tag
|
||||
tmpdir = tempfile.mkdtemp(prefix="tdd_consist_viol_")
|
||||
try:
|
||||
feature_file = Path(tmpdir) / "fake_violation.feature"
|
||||
feature_file.write_text(
|
||||
"@tdd_bug @tdd_bug_966 @tdd_expected_fail\n"
|
||||
"Feature: Fake violation\n"
|
||||
" Scenario: Dummy\n"
|
||||
" Given nothing\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Run the script, pointing --base-branch to a fake so that
|
||||
# git log returns our controlled commit message. We simulate
|
||||
# this by creating a tiny git repo with one commit referencing #966.
|
||||
git_dir = Path(tmpdir) / "gitrepo"
|
||||
git_dir.mkdir()
|
||||
env = {
|
||||
**os.environ,
|
||||
"GIT_AUTHOR_NAME": "test",
|
||||
"GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "test",
|
||||
"GIT_COMMITTER_EMAIL": "t@t",
|
||||
}
|
||||
subprocess.run(
|
||||
["git", "init", "--initial-branch=main", str(git_dir)],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
# Create initial commit on 'base' branch
|
||||
dummy = git_dir / "dummy.txt"
|
||||
dummy.write_text("base", encoding="utf-8")
|
||||
subprocess.run(
|
||||
["git", "-C", str(git_dir), "add", "."],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "-C", str(git_dir), "commit", "-m", "initial"],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
env=env,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "-C", str(git_dir), "branch", "base"],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
# Create a commit referencing #966
|
||||
dummy.write_text("fix", encoding="utf-8")
|
||||
subprocess.run(
|
||||
["git", "-C", str(git_dir), "add", "."],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "-C", str(git_dir), "commit", "-m", "fix: resolve #966"],
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
# Run the script in the git repo, scanning the temp feature file
|
||||
# We need to override _PROJECT_ROOT, so we use a wrapper approach:
|
||||
# run the script directly and point features/ to our temp dir.
|
||||
wrapper = git_dir / "run_check.py"
|
||||
wrapper.write_text(
|
||||
"import sys, importlib\n"
|
||||
f"sys.path.insert(0, {_SCRIPT!r}.rsplit('/', 1)[0])\n"
|
||||
"mod = importlib.import_module('check-tdd-tag-consistency')\n"
|
||||
"from pathlib import Path\n"
|
||||
f"mod._PROJECT_ROOT = Path({tmpdir!r})\n"
|
||||
"# Create minimal features/ dir structure for scanner\n"
|
||||
f"(Path({tmpdir!r}) / 'features').mkdir(exist_ok=True)\n"
|
||||
"import shutil\n"
|
||||
f"shutil.copy({str(feature_file)!r}, "
|
||||
f"str(Path({tmpdir!r}) / 'features' / 'fake_violation.feature'))\n"
|
||||
"sys.exit(mod.main(['--base-branch', 'base', "
|
||||
"'--forgejo-url', '']))\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(wrapper)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
cwd=str(git_dir),
|
||||
)
|
||||
|
||||
if proc.returncode != 1:
|
||||
print(
|
||||
f"FAIL: Expected rc=1 for violation but got "
|
||||
f"rc={proc.returncode}\n"
|
||||
f"stdout: {proc.stdout[:500]}\n"
|
||||
f"stderr: {proc.stderr[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if "FAILED" not in proc.stdout:
|
||||
print(
|
||||
f"FAIL: Expected 'FAILED' in output but got:\n{proc.stdout[:500]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("tdd-consistency-violation-exits-nonzero-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
_COMMANDS: dict[str, Callable[[], int]] = {
|
||||
"passes-on-master": cmd_passes_on_master,
|
||||
"reports-tag-count": cmd_reports_tag_count,
|
||||
"graceful-no-forgejo": cmd_graceful_no_forgejo,
|
||||
"violation-exits-nonzero": cmd_violation_exits_nonzero,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Dispatch to a sub-command based on the first CLI argument."""
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(
|
||||
f"Usage: {sys.argv[0]} <{'|'.join(sorted(_COMMANDS))}>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(2)
|
||||
handler = _COMMANDS[sys.argv[1]]
|
||||
sys.exit(handler())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for the TDD tag consistency checker script.
|
||||
... Verifies exit codes, output format, and graceful degradation.
|
||||
Library Process
|
||||
Resource common.resource
|
||||
Force Tags tdd_consistency integration v3.2.0
|
||||
|
||||
Suite Setup Setup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${PYTHON} python3
|
||||
${HELPER} ${CURDIR}/helper_tdd_consistency_check.py
|
||||
|
||||
*** Test Cases ***
|
||||
TDD Consistency Passes On Master
|
||||
[Documentation] The consistency check script should exit 0 on master
|
||||
... with no violations and print a success message.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} passes-on-master
|
||||
... timeout=120s on_timeout=kill
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
... msg=passes-on-master failed: ${result.stdout} ${result.stderr}
|
||||
|
||||
TDD Consistency Reports Tag Count
|
||||
[Documentation] The script should report how many @tdd_bug_<N> tags
|
||||
... it found across test suites.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} reports-tag-count
|
||||
... timeout=120s on_timeout=kill
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
... msg=reports-tag-count failed: ${result.stdout} ${result.stderr}
|
||||
|
||||
TDD Consistency Graceful Without Forgejo
|
||||
[Documentation] When --forgejo-url is empty, Check 2 should be
|
||||
... skipped gracefully (no error, just a skip message).
|
||||
${result}= Run Process ${PYTHON} ${HELPER} graceful-no-forgejo
|
||||
... timeout=120s on_timeout=kill
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
... msg=graceful-no-forgejo failed: ${result.stdout} ${result.stderr}
|
||||
|
||||
TDD Consistency Violation Exits Nonzero
|
||||
[Documentation] P1-3: The script must exit 1 when violations exist.
|
||||
... Creates a controlled scenario with a @tdd_expected_fail tag on
|
||||
... an issue referenced by a commit, then verifies rc=1 and FAILED
|
||||
... in output.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} violation-exits-nonzero
|
||||
... timeout=120s on_timeout=kill
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
... msg=violation-exits-nonzero failed: ${result.stdout} ${result.stderr}
|
||||
@@ -0,0 +1,522 @@
|
||||
#!/usr/bin/env python3
|
||||
"""TDD tag consistency checker — enforces @tdd_bug / @tdd_expected_fail rules.
|
||||
|
||||
Check 1 (local): commits referencing #N must remove @tdd_expected_fail
|
||||
from @tdd_bug_N tests. Check 2 (Forgejo API): tag state must match
|
||||
issue open/closed status. See CONTRIBUTING.md > TDD Bug Test Tags.
|
||||
|
||||
NOTE: Issue #966 uses @tdd_issue but the codebase uses @tdd_bug.
|
||||
Exit 0 = pass, 1 = violations found.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from json import JSONDecodeError
|
||||
from json import loads as json_loads
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
Suite = Literal["behave-unit", "robot-integration", "robot-e2e"]
|
||||
|
||||
_TDD_BUG_N_RE = re.compile(r"tdd_bug_(\d+)")
|
||||
|
||||
_ISSUE_REF_RE = re.compile(r"(?<!\w)#(\d+)\b")
|
||||
|
||||
# Synthetic issue numbers from fixture files; skip in Check 2.
|
||||
_SYNTHETIC_ISSUE_NUMBERS: frozenset[int] = frozenset({997, 998, 999})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TagInfo:
|
||||
"""A single @tdd_bug_<N> occurrence in a test file."""
|
||||
|
||||
file: Path
|
||||
line: int
|
||||
issue_number: int
|
||||
has_expected_fail: bool
|
||||
suite: Suite
|
||||
|
||||
|
||||
@dataclass
|
||||
class Violation:
|
||||
"""A consistency violation to report."""
|
||||
|
||||
file: Path
|
||||
line: int
|
||||
issue_number: int
|
||||
tag: str
|
||||
message: str
|
||||
suite: Suite
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
"""Aggregated result from one check."""
|
||||
|
||||
violations: list[Violation] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _scan_behave_features() -> list[TagInfo]:
|
||||
"""Scan ``features/**/*.feature`` for TDD bug tags."""
|
||||
results: list[TagInfo] = []
|
||||
features_dir = _PROJECT_ROOT / "features"
|
||||
for fpath in sorted(features_dir.rglob("*.feature")):
|
||||
rel = fpath.relative_to(_PROJECT_ROOT)
|
||||
if "testing" in rel.parts:
|
||||
continue
|
||||
_scan_behave_file(fpath, "behave-unit", results)
|
||||
return results
|
||||
|
||||
|
||||
def _scan_behave_file(fpath: Path, suite: Suite, results: list[TagInfo]) -> None:
|
||||
"""Parse a .feature file for TDD tags."""
|
||||
try:
|
||||
text = fpath.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
print(f" WARNING: skipping unreadable file: {fpath}", file=sys.stderr)
|
||||
return
|
||||
lines = text.splitlines()
|
||||
|
||||
feature_level_bugs: set[int] = set()
|
||||
feature_level_expected_fail = False
|
||||
in_feature_header = True
|
||||
|
||||
pending_scenario_bugs: set[int] = set()
|
||||
pending_scenario_ef = False
|
||||
|
||||
def _emit_scenario(at_line: int) -> None:
|
||||
all_bugs = feature_level_bugs | pending_scenario_bugs
|
||||
has_ef = feature_level_expected_fail or pending_scenario_ef
|
||||
for issue_num in all_bugs:
|
||||
results.append(
|
||||
TagInfo(
|
||||
file=fpath,
|
||||
line=at_line,
|
||||
issue_number=issue_num,
|
||||
has_expected_fail=has_ef,
|
||||
suite=suite,
|
||||
)
|
||||
)
|
||||
|
||||
for lineno_0, raw in enumerate(lines):
|
||||
lineno = lineno_0 + 1
|
||||
stripped = raw.strip()
|
||||
|
||||
if in_feature_header and stripped.startswith("Feature:"):
|
||||
in_feature_header = False
|
||||
continue
|
||||
|
||||
if in_feature_header and stripped.startswith("@"):
|
||||
for m in _TDD_BUG_N_RE.finditer(stripped):
|
||||
feature_level_bugs.add(int(m.group(1)))
|
||||
if "tdd_expected_fail" in stripped:
|
||||
feature_level_expected_fail = True
|
||||
continue
|
||||
|
||||
if in_feature_header:
|
||||
continue
|
||||
|
||||
if stripped.startswith("@"):
|
||||
for m in _TDD_BUG_N_RE.finditer(stripped):
|
||||
pending_scenario_bugs.add(int(m.group(1)))
|
||||
if "tdd_expected_fail" in stripped:
|
||||
pending_scenario_ef = True
|
||||
continue
|
||||
|
||||
if stripped.startswith("Background:"):
|
||||
pending_scenario_bugs = set()
|
||||
pending_scenario_ef = False
|
||||
continue
|
||||
|
||||
if (
|
||||
stripped.startswith("Scenario:")
|
||||
or stripped.startswith("Scenario Outline:")
|
||||
or stripped.startswith("Scenario Template:")
|
||||
):
|
||||
_emit_scenario(lineno)
|
||||
pending_scenario_bugs = set()
|
||||
pending_scenario_ef = False
|
||||
|
||||
|
||||
def _extract_tags_from_line(line: str) -> tuple[set[int], bool]:
|
||||
"""Extract tdd_bug_N numbers and tdd_expected_fail from a line."""
|
||||
bugs: set[int] = set()
|
||||
for m in _TDD_BUG_N_RE.finditer(line):
|
||||
bugs.add(int(m.group(1)))
|
||||
has_ef = "tdd_expected_fail" in line
|
||||
return bugs, has_ef
|
||||
|
||||
|
||||
def _scan_robot_files(
|
||||
directory: Path, suite: Suite, *, exclude_e2e: bool = False
|
||||
) -> list[TagInfo]:
|
||||
"""Scan ``.robot`` files for TDD bug tags."""
|
||||
results: list[TagInfo] = []
|
||||
for fpath in sorted(directory.rglob("*.robot")):
|
||||
rel = fpath.relative_to(_PROJECT_ROOT)
|
||||
if "fixtures" in rel.parts:
|
||||
continue
|
||||
if exclude_e2e and "e2e" in rel.parts:
|
||||
continue
|
||||
_scan_robot_file(fpath, suite, results)
|
||||
return results
|
||||
|
||||
|
||||
def _scan_robot_file(fpath: Path, suite: Suite, results: list[TagInfo]) -> None:
|
||||
"""Parse a .robot file for TDD tags."""
|
||||
try:
|
||||
text = fpath.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
print(f" WARNING: skipping unreadable file: {fpath}", file=sys.stderr)
|
||||
return
|
||||
lines = text.splitlines()
|
||||
|
||||
force_bugs: set[int] = set()
|
||||
force_expected_fail = False
|
||||
|
||||
in_test_cases = False
|
||||
current_test_line: int | None = None
|
||||
current_test_bugs: set[int] = set()
|
||||
current_test_ef = False
|
||||
|
||||
def _collect_continuations(start: int) -> str:
|
||||
"""Merge starting line with ... continuation lines."""
|
||||
combined = lines[start]
|
||||
idx = start + 1
|
||||
while idx < len(lines) and lines[idx].strip().startswith("..."):
|
||||
# Strip the leading ``...`` and append
|
||||
tail = lines[idx].strip()[3:].strip()
|
||||
combined += " " + tail
|
||||
idx += 1
|
||||
return combined
|
||||
|
||||
def _emit_current_test() -> None:
|
||||
if current_test_line is None:
|
||||
return
|
||||
all_bugs = force_bugs | current_test_bugs
|
||||
has_ef = force_expected_fail or current_test_ef
|
||||
for issue_num in all_bugs:
|
||||
results.append(
|
||||
TagInfo(
|
||||
file=fpath,
|
||||
line=current_test_line,
|
||||
issue_number=issue_num,
|
||||
has_expected_fail=has_ef,
|
||||
suite=suite,
|
||||
)
|
||||
)
|
||||
|
||||
for lineno_0, raw in enumerate(lines):
|
||||
lineno = lineno_0 + 1
|
||||
stripped = raw.strip()
|
||||
|
||||
if stripped.startswith("***") and stripped.endswith("***"):
|
||||
_emit_current_test()
|
||||
current_test_line = None
|
||||
current_test_bugs = set()
|
||||
current_test_ef = False
|
||||
in_test_cases = "test cases" in stripped.lower()
|
||||
continue
|
||||
|
||||
if stripped.startswith("Force Tags"):
|
||||
full_line = _collect_continuations(lineno_0)
|
||||
bugs, has_ef = _extract_tags_from_line(full_line)
|
||||
force_bugs.update(bugs)
|
||||
if has_ef:
|
||||
force_expected_fail = True
|
||||
continue
|
||||
|
||||
if not in_test_cases:
|
||||
continue
|
||||
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
|
||||
# New Robot test case header: non-indented line in *** Test Cases ***.
|
||||
if not raw.startswith((" ", "\t")) and not stripped.startswith(("...", "[")):
|
||||
_emit_current_test()
|
||||
current_test_line = lineno
|
||||
current_test_bugs = set()
|
||||
current_test_ef = False
|
||||
continue
|
||||
|
||||
if stripped.startswith("[Tags]"):
|
||||
full_line = _collect_continuations(lineno_0)
|
||||
test_bugs, test_ef = _extract_tags_from_line(full_line)
|
||||
current_test_bugs.update(test_bugs)
|
||||
if test_ef:
|
||||
current_test_ef = True
|
||||
|
||||
_emit_current_test()
|
||||
|
||||
|
||||
def scan_all_test_files() -> list[TagInfo]:
|
||||
"""Scan all three test suites and return aggregated tag info."""
|
||||
results: list[TagInfo] = []
|
||||
results.extend(_scan_behave_features())
|
||||
robot_dir = _PROJECT_ROOT / "robot"
|
||||
if robot_dir.is_dir():
|
||||
results.extend(
|
||||
_scan_robot_files(robot_dir, "robot-integration", exclude_e2e=True)
|
||||
)
|
||||
e2e_dir = robot_dir / "e2e"
|
||||
if e2e_dir.is_dir():
|
||||
results.extend(_scan_robot_files(e2e_dir, "robot-e2e"))
|
||||
return results
|
||||
|
||||
|
||||
def get_branch_issue_references(base_branch: str) -> tuple[set[int], str | None]:
|
||||
"""Extract ``#N`` issue references from commit messages on current branch."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "log", "--format=%B", f"{base_branch}..HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
except (
|
||||
subprocess.CalledProcessError,
|
||||
subprocess.TimeoutExpired,
|
||||
FileNotFoundError,
|
||||
) as exc:
|
||||
return set(), f"git log failed ({type(exc).__name__})"
|
||||
|
||||
refs: set[int] = set()
|
||||
for m in _ISSUE_REF_RE.finditer(result.stdout):
|
||||
refs.add(int(m.group(1)))
|
||||
return refs, None
|
||||
|
||||
|
||||
def check_commit_consistency(base_branch: str, tag_infos: list[TagInfo]) -> CheckResult:
|
||||
"""Check 1: commits reference issue N but tdd_expected_fail still present."""
|
||||
result = CheckResult()
|
||||
referenced, warning = get_branch_issue_references(base_branch)
|
||||
if warning:
|
||||
result.warnings.append(warning)
|
||||
if not referenced:
|
||||
return result
|
||||
|
||||
for info in tag_infos:
|
||||
if info.issue_number in referenced and info.has_expected_fail:
|
||||
result.violations.append(
|
||||
Violation(
|
||||
file=info.file,
|
||||
line=info.line,
|
||||
issue_number=info.issue_number,
|
||||
tag=f"@tdd_bug_{info.issue_number}",
|
||||
message=(
|
||||
f"Commit references #{info.issue_number} but "
|
||||
f"@tdd_expected_fail is still present. Remove "
|
||||
f"@tdd_expected_fail — the issue is being closed."
|
||||
),
|
||||
suite=info.suite,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
_ISSUE_NOT_FOUND = "not_found"
|
||||
|
||||
|
||||
def _fetch_issue_state(forgejo_url: str, issue_number: int) -> str | None:
|
||||
"""Query Forgejo API for issue state using FORGEJO_TOKEN from env."""
|
||||
parsed = urllib.parse.urlparse(forgejo_url)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
return None
|
||||
url = (
|
||||
f"{forgejo_url.rstrip('/')}/api/v1/repos/"
|
||||
f"cleveragents/cleveragents-core/issues/{issue_number}"
|
||||
)
|
||||
req = urllib.request.Request(url)
|
||||
effective_token = os.environ.get("FORGEJO_TOKEN", "")
|
||||
if effective_token:
|
||||
req.add_header("Authorization", f"token {effective_token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json_loads(resp.read().decode("utf-8"))
|
||||
state = data.get("state")
|
||||
if isinstance(state, str):
|
||||
return state
|
||||
return None
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return _ISSUE_NOT_FOUND
|
||||
return None
|
||||
except (urllib.error.URLError, OSError, JSONDecodeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def check_forgejo_consistency(
|
||||
forgejo_url: str,
|
||||
tag_infos: list[TagInfo],
|
||||
) -> CheckResult:
|
||||
"""Check 2: cross-reference tag state with Forgejo issue status."""
|
||||
result = CheckResult()
|
||||
|
||||
if not forgejo_url:
|
||||
result.warnings.append(
|
||||
"Skipping Forgejo API check: --forgejo-url not provided."
|
||||
)
|
||||
return result
|
||||
|
||||
issue_numbers: set[int] = set()
|
||||
for info in tag_infos:
|
||||
if info.issue_number not in _SYNTHETIC_ISSUE_NUMBERS:
|
||||
issue_numbers.add(info.issue_number)
|
||||
|
||||
issue_states: dict[int, str] = {}
|
||||
api_reachable = True
|
||||
for num in sorted(issue_numbers):
|
||||
state = _fetch_issue_state(forgejo_url, num)
|
||||
if state == _ISSUE_NOT_FOUND:
|
||||
result.warnings.append(f"Issue #{num} not found (HTTP 404) — skipping.")
|
||||
continue
|
||||
if state is None:
|
||||
api_reachable = False
|
||||
result.warnings.append(
|
||||
f"Forgejo API error for issue #{num}. "
|
||||
f"Remaining issues will be skipped (API may be unreachable)."
|
||||
)
|
||||
break
|
||||
issue_states[num] = state
|
||||
|
||||
if not api_reachable:
|
||||
result.warnings.append(
|
||||
"Forgejo API check degraded: some issues could not be queried."
|
||||
)
|
||||
return result
|
||||
|
||||
for info in tag_infos:
|
||||
if info.issue_number in _SYNTHETIC_ISSUE_NUMBERS:
|
||||
continue
|
||||
state = issue_states.get(info.issue_number)
|
||||
if state is None:
|
||||
continue
|
||||
|
||||
if info.has_expected_fail and state == "closed":
|
||||
result.violations.append(
|
||||
Violation(
|
||||
file=info.file,
|
||||
line=info.line,
|
||||
issue_number=info.issue_number,
|
||||
tag=f"@tdd_bug_{info.issue_number}",
|
||||
message=(
|
||||
f"Issue #{info.issue_number} is CLOSED but "
|
||||
f"@tdd_expected_fail is still present. Remove "
|
||||
f"@tdd_expected_fail — the issue is resolved."
|
||||
),
|
||||
suite=info.suite,
|
||||
)
|
||||
)
|
||||
elif not info.has_expected_fail and state == "open":
|
||||
result.violations.append(
|
||||
Violation(
|
||||
file=info.file,
|
||||
line=info.line,
|
||||
issue_number=info.issue_number,
|
||||
tag=f"@tdd_bug_{info.issue_number}",
|
||||
message=(
|
||||
f"Issue #{info.issue_number} is OPEN but "
|
||||
f"@tdd_expected_fail is absent. Add "
|
||||
f"@tdd_expected_fail — the issue is unresolved."
|
||||
),
|
||||
suite=info.suite,
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _format_violations(violations: list[Violation]) -> str:
|
||||
"""Format violations for human-readable output."""
|
||||
if not violations:
|
||||
return ""
|
||||
out: list[str] = []
|
||||
for v in violations:
|
||||
rel = v.file.relative_to(_PROJECT_ROOT)
|
||||
out.append(f" ERROR [{v.suite}] {rel}:{v.line} {v.tag}: {v.message}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Run TDD tag consistency checks. Returns 0 on success, 1 on failure."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check TDD tag consistency across test suites."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--forgejo-url",
|
||||
default=os.environ.get("FORGEJO_URL", ""),
|
||||
help="Forgejo URL (env: FORGEJO_URL).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-branch",
|
||||
default=os.environ.get("TDD_BASE_BRANCH", "origin/master"),
|
||||
help="Base branch (env: TDD_BASE_BRANCH).",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
print("=== TDD Tag Consistency Check ===\n")
|
||||
|
||||
print("Scanning test files...")
|
||||
tag_infos = scan_all_test_files()
|
||||
issue_count = len({i.issue_number for i in tag_infos})
|
||||
print(
|
||||
f" Found {len(tag_infos)} @tdd_bug_<N> tags "
|
||||
f"referencing {issue_count} issues.\n"
|
||||
)
|
||||
|
||||
all_violations: list[Violation] = []
|
||||
all_warnings: list[str] = []
|
||||
|
||||
print(f"Check 1: Commit consistency (base: {args.base_branch})...")
|
||||
c1 = check_commit_consistency(args.base_branch, tag_infos)
|
||||
all_violations.extend(c1.violations)
|
||||
all_warnings.extend(c1.warnings)
|
||||
if c1.violations:
|
||||
print(f" {len(c1.violations)} violation(s) found.")
|
||||
else:
|
||||
print(" PASS — no violations.")
|
||||
print()
|
||||
|
||||
print("Check 2: Forgejo issue-status consistency...")
|
||||
c2 = check_forgejo_consistency(args.forgejo_url, tag_infos)
|
||||
all_violations.extend(c2.violations)
|
||||
all_warnings.extend(c2.warnings)
|
||||
if c2.violations:
|
||||
print(f" {len(c2.violations)} violation(s) found.")
|
||||
elif c2.warnings:
|
||||
print(f" WARN — {len(c2.warnings)} warning(s).")
|
||||
else:
|
||||
print(" PASS — no violations.")
|
||||
print()
|
||||
|
||||
for w in all_warnings:
|
||||
print(f" WARNING: {w}")
|
||||
|
||||
if all_violations:
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"FAILED: {len(all_violations)} TDD tag consistency violation(s):\n")
|
||||
print(_format_violations(all_violations))
|
||||
print(f"\n{'=' * 60}")
|
||||
return 1
|
||||
|
||||
print("All TDD tag consistency checks passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user