Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4c419d48f5 | |||
| daee737934 | |||
| d1328e562f | |||
| b4a514e0f4 | |||
|
e8d2f76466
|
|||
| dd763f50d9 | |||
| 81229422f2 |
@@ -15,9 +15,6 @@ on:
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -29,7 +26,7 @@ jobs:
|
||||
runs-on: docker
|
||||
timeout-minutes: 120
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
@@ -114,7 +111,7 @@ jobs:
|
||||
runs-on: docker
|
||||
timeout-minutes: 180
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
|
||||
+61
-39
@@ -6,9 +6,6 @@ on:
|
||||
pull_request:
|
||||
branches: [master, develop*]
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -18,7 +15,7 @@ jobs:
|
||||
lint:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -62,7 +59,7 @@ jobs:
|
||||
typecheck:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -100,7 +97,7 @@ jobs:
|
||||
security:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -144,7 +141,7 @@ jobs:
|
||||
quality:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -182,7 +179,7 @@ jobs:
|
||||
unit_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests, curl/tar for Helm)
|
||||
run: |
|
||||
@@ -233,7 +230,7 @@ jobs:
|
||||
integration_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for integration tests, curl/tar for Helm)
|
||||
run: |
|
||||
@@ -288,11 +285,48 @@ jobs:
|
||||
path: build/nox-integration-tests-output.log
|
||||
retention-days: 30
|
||||
|
||||
tdd_quality_gate:
|
||||
if: forgejo.event_name == 'pull_request'
|
||||
runs-on: docker
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for diff analysis)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Fetch PR base branch for diff analysis
|
||||
run: |
|
||||
git fetch origin "${{ forgejo.base_ref }}"
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-tdd-gate-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-tdd-gate-
|
||||
|
||||
- name: Run TDD quality gate via nox
|
||||
run: |
|
||||
nox -s tdd_quality_gate
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
PR_DESCRIPTION: ${{ forgejo.event.pull_request.body }}
|
||||
PR_BASE_REF: ${{ forgejo.base_ref }}
|
||||
|
||||
e2e_tests:
|
||||
runs-on: docker
|
||||
timeout-minutes: 45
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for E2E tests)
|
||||
run: |
|
||||
@@ -308,40 +342,23 @@ jobs:
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ hashFiles('pyproject.toml') }}
|
||||
key: uv-tests-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-
|
||||
uv-tests-
|
||||
|
||||
- name: Run E2E tests via nox
|
||||
run: |
|
||||
mkdir -p build
|
||||
nox -s e2e_tests 2>&1 | tee build/nox-e2e-tests-output.log
|
||||
nox -s e2e_tests
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
# Run E2E suites in parallel via pabot. 4 workers keeps
|
||||
# wall-clock time well under the 45-minute timeout while
|
||||
# staying within the memory budget of the docker runner.
|
||||
TEST_PROCESSES: "4"
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
- name: Upload E2E tests log artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-logs-e2e-tests
|
||||
path: |
|
||||
build/nox-e2e-tests-output.log
|
||||
build/reports/robot-e2e/
|
||||
retention-days: 30
|
||||
|
||||
coverage:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
# unit_tests is included so coverage only runs after tests pass,
|
||||
# preventing misleading results when tests are still in-flight or failing.
|
||||
image: python:3.13-slim
|
||||
needs: [lint, typecheck, security, quality, unit_tests]
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
@@ -416,7 +433,7 @@ jobs:
|
||||
build:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -472,7 +489,7 @@ jobs:
|
||||
helm:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, curl for Helm and kubeconform)
|
||||
run: |
|
||||
@@ -537,7 +554,7 @@ jobs:
|
||||
# config (name/email) was set — both are required for any push operation.
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for push validation)
|
||||
run: |
|
||||
@@ -610,10 +627,10 @@ jobs:
|
||||
echo "=== Push access smoke-test passed ==="
|
||||
status-check:
|
||||
if: always()
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, e2e_tests, coverage, build, docker, helm, push-validation]
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, coverage, build, docker, helm, push-validation, tdd_quality_gate]
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Check required job results
|
||||
run: |
|
||||
@@ -623,12 +640,12 @@ jobs:
|
||||
echo "quality: ${{ needs.quality.result }}"
|
||||
echo "unit_tests: ${{ needs.unit_tests.result }}"
|
||||
echo "integration_tests: ${{ needs.integration_tests.result }}"
|
||||
echo "e2e_tests: ${{ needs.e2e_tests.result }}"
|
||||
echo "coverage: ${{ needs.coverage.result }}"
|
||||
echo "build: ${{ needs.build.result }}"
|
||||
echo "docker: ${{ needs.docker.result }}"
|
||||
echo "helm: ${{ needs.helm.result }}"
|
||||
echo "push-validation: ${{ needs.push-validation.result }}"
|
||||
echo "tdd_quality_gate: ${{ needs.tdd_quality_gate.result }}"
|
||||
|
||||
if [ "${{ needs.lint.result }}" != "success" ] || \
|
||||
[ "${{ needs.typecheck.result }}" != "success" ] || \
|
||||
@@ -636,7 +653,6 @@ jobs:
|
||||
[ "${{ needs.quality.result }}" != "success" ] || \
|
||||
[ "${{ needs.unit_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.integration_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.e2e_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.coverage.result }}" != "success" ] || \
|
||||
[ "${{ needs.build.result }}" != "success" ] || \
|
||||
[ "${{ needs.docker.result }}" != "success" ] || \
|
||||
@@ -645,4 +661,10 @@ jobs:
|
||||
echo "FAILED: One or more required jobs did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ forgejo.event_name }}" = "pull_request" ] && \
|
||||
[ "${{ needs.tdd_quality_gate.result }}" != "success" ]; then
|
||||
echo "FAILED: tdd_quality_gate did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
echo "All required CI checks passed"
|
||||
|
||||
@@ -3,11 +3,6 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [master, develop]
|
||||
pull_request:
|
||||
branches: [master, develop]
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
@@ -15,10 +10,67 @@ env:
|
||||
NOX_DEFAULT_VENV_BACKEND: "uv"
|
||||
|
||||
jobs:
|
||||
benchmark-regression:
|
||||
if: forgejo.event_name != 'pull_request'
|
||||
runs-on: docker-benchmark
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- name: Checkout full history
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Compute base commit
|
||||
id: hash
|
||||
run: |
|
||||
git fetch origin "${{ forgejo.base_ref }}" --depth=200
|
||||
BASE_SHA=$(git merge-base HEAD "origin/${{ forgejo.base_ref }}")
|
||||
echo "ASV_BASE_SHA=${BASE_SHA}" >> $FORGEJO_OUTPUT
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install -U pip
|
||||
python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Restore prior ASV benchmarks
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
||||
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
||||
run: |
|
||||
python -m pip install awscli
|
||||
mkdir -p build/asv/results
|
||||
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true
|
||||
|
||||
- name: Run asv continuous via nox
|
||||
env:
|
||||
ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }}
|
||||
run: |
|
||||
nox -s benchmark_regression
|
||||
|
||||
- name: Archive the results
|
||||
run: |
|
||||
tar cf /tmp/asv-results.tar build/asv/results build/asv/html
|
||||
|
||||
- name: Upload benchmark artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: asv-results-pr
|
||||
path: /tmp/asv-results.tar
|
||||
retention-days: 30
|
||||
|
||||
benchmark-publish:
|
||||
if: forgejo.event_name == 'push'
|
||||
runs-on: docker-benchmark
|
||||
container: ${{vars.docker_prefix}}python:3.13-slim
|
||||
container: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
run: |
|
||||
@@ -71,56 +123,51 @@ jobs:
|
||||
name: asv-results-pr
|
||||
path: /tmp/asv-results.tar
|
||||
retention-days: 30
|
||||
|
||||
benchmark-regression:
|
||||
# Runs benchmark regression on pull requests to detect performance regressions.
|
||||
# This job is informational only — it is NOT listed in status-check's required
|
||||
# needs, so a benchmark regression does not block PR merges.
|
||||
if: forgejo.event_name == 'pull_request'
|
||||
runs-on: docker-benchmark
|
||||
container: ${{vars.docker_prefix}}python:3.13-slim
|
||||
e2e_tests:
|
||||
if: forgejo.event_name == 'push'
|
||||
runs-on: docker
|
||||
timeout-minutes: 45
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for ASV)
|
||||
- name: Install system dependencies (nodejs for checkout, git for E2E tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/*
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- name: Checkout full history
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install -U pip
|
||||
python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Sync prior benchmark results from S3
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
||||
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
||||
run: |
|
||||
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
||||
python -m pip install awscli
|
||||
mkdir -p build/asv/results
|
||||
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results/" build/asv/results/ || echo "No existing results to sync"
|
||||
else
|
||||
echo "Skipping S3 sync - AWS credentials not configured"
|
||||
fi
|
||||
|
||||
- name: Run benchmark regression via nox
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
ASV_BASE_SHA: master
|
||||
- name: Run E2E tests via nox
|
||||
run: |
|
||||
mkdir -p build
|
||||
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-regression-output.log
|
||||
nox -s e2e_tests 2>&1 | tee build/nox-e2e-tests-output.log
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
# Run E2E suites in parallel via pabot. 4 workers keeps
|
||||
# wall-clock time well under the 45-minute timeout while
|
||||
# staying within the memory budget of the docker runner.
|
||||
TEST_PROCESSES: "4"
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
- name: Upload benchmark regression log artifact
|
||||
if: always()
|
||||
- name: Upload E2E tests log artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: benchmark-regression-logs
|
||||
path: build/nox-benchmark-regression-output.log
|
||||
retention-days: 30
|
||||
name: ci-logs-e2e-tests
|
||||
path: |
|
||||
build/nox-e2e-tests-output.log
|
||||
build/reports/robot-e2e/
|
||||
retention-days: 30
|
||||
@@ -7,9 +7,6 @@ on:
|
||||
workflow_dispatch:
|
||||
# Allow manual trigger for testing
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -19,7 +16,7 @@ jobs:
|
||||
full-quality-suite:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (git for merge tests)
|
||||
run: |
|
||||
|
||||
@@ -5,9 +5,6 @@ on:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -17,7 +14,7 @@ jobs:
|
||||
build-wheel:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -88,7 +85,7 @@ jobs:
|
||||
create-release:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
needs: [build-wheel, build-docker]
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
|
||||
+51
-2
@@ -5,6 +5,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- **`task-implementor` posts work-started notification comments** (#11031): Both
|
||||
the `issue_impl` and `pr_fix` procedures now post an informational "work
|
||||
started" comment to the Forgejo issue/PR before beginning implementation.
|
||||
The comment includes the issue/PR title, procedure type, and expected
|
||||
duration. Posted asynchronously ("fire and move on") so it does not block
|
||||
the workflow. Step numbering in both procedures has been re-numbered to
|
||||
accommodate the new step.
|
||||
|
||||
- **`agents session tell` invokes real LLM orchestrator actor** (#5784): Replaced the
|
||||
M3 echo-stub with real actor invocation via `SessionWorkflow`, routing through
|
||||
`LangChainSessionCaller` → `ToolCallingRuntime.run_tool_loop()`. The user prompt
|
||||
@@ -26,9 +34,41 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Added
|
||||
|
||||
- **`pr-review-worker` review-started notification** (#11028): The `first_review`
|
||||
and `re_review` modes now post a "review started" notification comment to the
|
||||
PR at the beginning of the review, giving PR authors immediate visibility
|
||||
that a review is in progress. Posted asynchronously so it does not block
|
||||
the workflow.
|
||||
|
||||
- **Plan Rollback Command** (#8557): Implemented `agents plan rollback <plan-id> [<checkpoint-id>]` for checkpoint-based plan state restoration in Epic #8493. The command restores a plan's sandbox to the state captured at a given checkpoint, discarding all decisions made after that checkpoint. The checkpoint can be specified as an optional positional second argument or via the `--to-checkpoint` named option. Supports `--yes/-y` flag to skip confirmation prompts and `--format/-f` for output format selection (rich/plain/json/yaml). Included with comprehensive BDD test coverage (>= 97%) and spec-aligned output formatting showing rollback summary, changes reverted, impact analysis, and post-rollback state panels.
|
||||
|
||||
### Fixed
|
||||
- Python 3.13 asyncio compatibility (issue #11134): replaced all deprecated ``asyncio.get_event_loop()`` calls with ``get_running_loop()`` + exception-safe fallback, updated ``datetime.utcnow()`` usage to ``datetime.now(timezone.utc)``, and added ``PYTHONWARNINGS=ignore::DeprecationWarning`` safety net in nox session. Fixes seven failing BDD scenarios caused by Python 3.13 deprecation warnings becoming errors during test execution across base.py, nodes.py, bridge.py, route_bridge.py, reactive_application_coverage_steps.py, langgraph_bridge_steps.py, and route_bridge_coverage_steps.py.
|
||||
- **Guard cleanup_stale against execute/processing and execute/complete plans** (#11121):
|
||||
``_create_sandbox_for_plan()`` in ``src/cleveragents/cli/commands/plan.py`` now
|
||||
skips ``GitWorktreeSandbox.cleanup_stale()`` when the plan is in
|
||||
``execute/processing`` (execution in progress) or ``execute/complete`` (execution
|
||||
finished, awaiting apply) state. Previously, re-invoking ``agents plan execute``
|
||||
on a completed plan would silently destroy the ``cleveragents/plan-<id>`` git
|
||||
worktree branch, causing ``plan apply`` to merge zero artifacts. The guard
|
||||
preserves the branch per spec (§sandbox.cleanup defaults to ``on_apply``).
|
||||
|
||||
- **Global CLI options ``--data-dir``, ``--config-path``, and ``-v`` now work correctly**
|
||||
(#6785): These spec-required flags were absent from ``main_callback()`` in
|
||||
``src/cleveragents/cli/main.py``, causing any invocation with these flags to crash
|
||||
with ``NoSuchOption: No such option``. All three options are now implemented per
|
||||
ADR-021 §Global CLI Flags and ADR-024 §Resolution Chain. ``--data-dir`` sets
|
||||
``CLEVERAGENTS_DATA_DIR`` before any ``Settings``-reading code runs, ``--config-path``
|
||||
sets ``CLEVERAGENTS_CONFIG_PATH`` for ``ConfigService`` to pick up, and ``-v``
|
||||
(repeatable count) maps to the appropriate ``structlog`` log level.
|
||||
|
||||
- **Add regression test for ActorRegistry.add() nested provider/model extraction** (#4321):
|
||||
Added BDD regression test confirming that provider and model are correctly
|
||||
extracted from nested `actors.<name>.config` blocks when `type` is only at
|
||||
the nested level and `name` uses the `local/` namespace prefix. This
|
||||
scenario was previously fixed in #4300; the test ensures the fix remains
|
||||
in place and prevents future regressions.
|
||||
|
||||
- **Actor configuration validation incorrectly requires top-level provider field** (#4300):
|
||||
Actor configuration in V3 is now obtained from the nested configuration
|
||||
parameter, according to the specification.
|
||||
@@ -270,6 +310,16 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`invariant_enforced` decisions not propagated to child plans on subplan spawn** (#9131):
|
||||
Fixed `SubplanService.spawn()` to propagate all `invariant_enforced` decisions from the
|
||||
parent plan's decision tree to each child plan's decision tree. Previously, child plans
|
||||
started Strategize with a completely empty invariant set, violating the spec requirement:
|
||||
"recorded as `invariant_enforced` decisions that propagate to child plans." The fix adds
|
||||
a `_propagate_invariant_decisions()` helper that re-records each parent
|
||||
`invariant_enforced` decision on the child plan, including `non_overridable` global
|
||||
invariants. BDD regression coverage added in
|
||||
`features/tdd_invariant_propagation_subplan.feature`.
|
||||
|
||||
- **fix(repositories): derive PlanResult.success from result_success column instead of error_message** (#7501):
|
||||
Fixed a critical bug in `PlanRepository._to_domain` where `PlanResult.success` was incorrectly
|
||||
derived from `error_message is None`. Because `error_message` is shared between the build phase
|
||||
@@ -858,5 +908,4 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
- **TUI -- Permission Question Widget**: A new inline `PermissionQuestionWidget`
|
||||
renders permission requests directly in the conversation stream for single-file
|
||||
operations. Users can allow/reject with single-key shortcuts (`a`/`A`/`r`/`R`),
|
||||
navigate with arrow keys, confirm with `Enter`, or press `v` to open the full
|
||||
|
||||
navigate with arrow keys, confirm with `Enter`, or press `v` to open the full
|
||||
@@ -1236,6 +1236,7 @@ session is missing required tooling, add the dependency to the session before re
|
||||
- **All tests (including static checks):** `nox`
|
||||
- **Unit tests only:** `nox -s unit_tests`
|
||||
- **Integration tests:** `nox -s integration_tests`
|
||||
- **TDD bug-fix quality gate:** `PR_DESCRIPTION="Fixes #123" PR_BASE_REF=master nox -s tdd_quality_gate`
|
||||
- **Benchmarks:** `nox -s benchmark`
|
||||
- **Coverage report:** `nox -s coverage_report`
|
||||
|
||||
@@ -1274,6 +1275,7 @@ The CI pipeline runs the following jobs. Jobs without explicit dependencies run
|
||||
| `quality` | — | Radon complexity analysis via `nox -s complexity` |
|
||||
| `unit_tests` | — | Behave BDD tests via `nox -s unit_tests` |
|
||||
| `integration_tests` | — | Robot Framework integration tests via `nox -s integration_tests` |
|
||||
| `tdd_quality_gate` | — (PR-only) | TDD bug-fix workflow enforcement via `nox -s tdd_quality_gate` (checks bug refs, TDD tags, and expected-fail tag removal in PR diff) |
|
||||
| `e2e_tests` | — | End-to-end Robot tests with real LLM keys via `nox -s e2e_tests` |
|
||||
| `coverage` | `lint`, `typecheck` | Slipcover coverage report via `nox -s coverage_report` (fail-under 97%) |
|
||||
| `benchmark-regression` | `lint`, `typecheck` | ASV benchmark regression on PRs via `nox -s benchmark_regression` |
|
||||
@@ -1293,6 +1295,7 @@ mergeable:
|
||||
- **security** — No high-severity Bandit findings, Semgrep rules pass, no dead code
|
||||
- **unit_tests** — All Behave BDD scenarios pass
|
||||
- **coverage** — Test coverage >= 97%
|
||||
- **tdd_quality_gate** — On PRs only, bug-fix PRs must satisfy the TDD Bug Fix Workflow gate
|
||||
|
||||
#### How to Read CI Results
|
||||
|
||||
|
||||
+6
-1
@@ -8,6 +8,12 @@
|
||||
* Luis Mendes <luis.p.mendes@gmail.com>
|
||||
* Rui Hu <rui.hu@cleverthis.com>
|
||||
|
||||
## Contributors
|
||||
|
||||
### Code Contributions
|
||||
|
||||
- **[asyncio and Deprecation Fixes](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/11134)** - Replaced deprecated `asyncio.get_event_loop()` calls with `get_running_loop()` + safe fallback, updated `datetime.utcnow()` usage to `datetime.now(timezone.utc)`, and added PYTHONWARNINGS safety net in nox session. Fixed seven failing BDD scenarios under Python 3.13 (issue #11134).
|
||||
|
||||
# Details
|
||||
|
||||
Below are some of the specific details of various contributions.
|
||||
@@ -17,7 +23,6 @@ Below are some of the specific details of various contributions.
|
||||
* HAL 9000 has contributed automated implementation, bug fixes, and feature development as part of the CleverAgents automation pool.
|
||||
* HAL 9000 has contributed concurrency safety improvements, including thread-safe context tier management (issue #7547) for parallel plan execution.
|
||||
* HAL 9000 has contributed the plan concurrency race-condition fix (#7989): wired `LockService` into the plan lifecycle, guarding `execute_plan()` and `apply_plan()` with plan-level advisory locks and unique per-invocation owner identities to prevent silent concurrent state corruption.
|
||||
<<<<<<< HEAD
|
||||
* HAL 9000 has contributed the bug-hunt-pool-supervisor non-blocking tracking fix (#7875 / PR #7957): updated step 5 to be best-effort and added rule 9 to prevent the automation-tracking-manager call from blocking the main supervisor loop.
|
||||
* Jeffrey Phillips Freeman has contributed the complete AUTO-BUG-POOL to AUTO-BUG-SUP tracking prefix fix across agent-system-specification.md, automation-tracking.md documentation and agent-system-specification.md spec document, replaced with correct `AUTO-BUG-SUP` prefix used by the bug-hunt-pool-supervisor agent (#7875).
|
||||
* HAL 9000 has contributed the plugin entry point security hardening fix (#7476): enforced entry point allowlist validation before importing plugin modules to prevent malicious plugin loading.
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
Feature: CLI global options --data-dir, --config-path, and -v
|
||||
|
||||
Background:
|
||||
Given global options test env is clean
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --data-dir option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: --data-dir option is accepted with an existing directory
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: --data-dir option overrides CLEVERAGENTS_DATA_DIR for the invocation
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
|
||||
Scenario: --data-dir with a path that is an existing file fails with a clear error
|
||||
Given a temp file path is prepared
|
||||
When I run the global options CLI with data-dir as temp file and "version"
|
||||
Then the global options CLI should fail
|
||||
And the global options output should contain "--data-dir"
|
||||
|
||||
Scenario: --data-dir stores value in ctx.obj for subcommands
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --config-path option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: --config-path option is accepted with an existing file
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: --config-path overrides CLEVERAGENTS_CONFIG_PATH for the invocation
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
Scenario: --config-path with a non-existent file fails with a clear error
|
||||
When I run the global options CLI with invalid config-path and "version"
|
||||
Then the global options CLI should fail
|
||||
And the global options output should contain "--config-path"
|
||||
|
||||
Scenario: --config-path stores value in ctx.obj for subcommands
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Combining --data-dir and --config-path
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: --data-dir and --config-path can be combined
|
||||
Given a temp data dir is prepared
|
||||
And a temp config file is prepared
|
||||
When I run the global options CLI with both path flags and "version"
|
||||
Then the global options CLI should succeed
|
||||
And the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
And the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# -v verbosity option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: No -v flag results in CRITICAL (silent) log level
|
||||
When I call main_callback with verbosity 0
|
||||
Then the global options log level should be "CRITICAL"
|
||||
|
||||
Scenario: Single -v flag results in ERROR log level
|
||||
When I call main_callback with verbosity 1
|
||||
Then the global options log level should be "ERROR"
|
||||
|
||||
Scenario: -vv results in WARNING log level
|
||||
When I call main_callback with verbosity 2
|
||||
Then the global options log level should be "WARNING"
|
||||
|
||||
Scenario: -vvv results in INFO log level
|
||||
When I call main_callback with verbosity 3
|
||||
Then the global options log level should be "INFO"
|
||||
|
||||
Scenario: -vvvv results in DEBUG log level
|
||||
When I call main_callback with verbosity 4
|
||||
Then the global options log level should be "DEBUG"
|
||||
|
||||
Scenario: -vvvvv or more results in DEBUG log level
|
||||
When I call main_callback with verbosity 5
|
||||
Then the global options log level should be "DEBUG"
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: -v flag is accepted by the CLI without crashing
|
||||
When I run the global options CLI with args "-v version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: -vvv flags are accepted by the CLI without crashing
|
||||
When I run the global options CLI with args "-v -v -v version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --help shows all three options
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: --help output includes --data-dir option
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "--data-dir"
|
||||
|
||||
Scenario: --help output includes --config-path option
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "--config-path"
|
||||
|
||||
Scenario: -v flag appears in --help output
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "-v"
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# -v verbosity stored in ctx.obj
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: verbose count is stored in ctx.obj
|
||||
When I call main_callback with verbosity 3
|
||||
Then the global options ctx obj "verbose" should be 3
|
||||
+15
-5
@@ -699,11 +699,21 @@ def after_scenario(context, scenario):
|
||||
pass # Ignore cleanup errors
|
||||
context.test_dir = None
|
||||
|
||||
# Clean up TemporaryDirectory objects created by ACMS index traversal tests
|
||||
if hasattr(context, "temp_dir") and context.temp_dir is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
context.temp_dir.cleanup()
|
||||
context.temp_dir = None
|
||||
# Clean up temp directories created by tdd_quality_gate and other steps.
|
||||
# temp_dir may be a Path, str, or tempfile.TemporaryDirectory depending
|
||||
# on which step file set it; handle all three safely.
|
||||
# NOTE: do NOT set context.temp_dir = None here — cleanup functions
|
||||
# registered via context.add_cleanup() run AFTER after_scenario and may
|
||||
# still reference context.temp_dir (e.g. cli_init_yes_flag_steps.py).
|
||||
# The scenario context layer is popped after all cleanups finish, which
|
||||
# removes the attribute automatically.
|
||||
temp_dir = getattr(context, "temp_dir", None)
|
||||
if temp_dir is not None:
|
||||
if isinstance(temp_dir, (str, Path)):
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
elif hasattr(temp_dir, "cleanup"):
|
||||
with contextlib.suppress(Exception):
|
||||
temp_dir.cleanup()
|
||||
|
||||
# Clean up environment variables set during tests
|
||||
if hasattr(context, "env_vars_to_clean"):
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Step definitions for CLI global options: --data-dir, --config-path, -v.
|
||||
|
||||
Tests for issue #6785 — spec-required global options that were missing from
|
||||
main_callback() and caused fatal "No such option" crashes.
|
||||
|
||||
All step patterns use the prefix "global options" to avoid any ambiguity
|
||||
with the many generic CLI step definitions in other step files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.formatting import OutputFormat
|
||||
from cleveragents.cli.main import app
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Regex to strip ANSI escape sequences (CSI sequences and OSC sequences).
|
||||
# Used to make output assertions environment-agnostic — Rich may emit ANSI
|
||||
# colour/style codes when the CI environment does not set TERM=dumb.
|
||||
_ANSI_ESCAPE_RE = re.compile(
|
||||
r"\x1b(?:"
|
||||
r"\[[0-9;]*[a-zA-Z]" # CSI sequences (SGR, cursor, etc.)
|
||||
r"|"
|
||||
r"\][^\x07]*\x07" # OSC sequences (hyperlink, title, etc.)
|
||||
r")"
|
||||
)
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
"""Remove ANSI escape sequences from *text*."""
|
||||
return _ANSI_ESCAPE_RE.sub("", text)
|
||||
|
||||
|
||||
def _invoke(args: list[str]) -> Any:
|
||||
"""Invoke the Typer app with the given args and return the result."""
|
||||
runner = CliRunner()
|
||||
return runner.invoke(app, args, catch_exceptions=True)
|
||||
|
||||
|
||||
def _combined_output(result: Any) -> str:
|
||||
"""Return combined output from a CliRunner result, with ANSI codes stripped.
|
||||
|
||||
In non-TTY environments (CI), Typer/Rich may split output between
|
||||
stdout and stderr. Check both explicitly so that ``--help`` output
|
||||
assertions work regardless of the terminal detection.
|
||||
"""
|
||||
stdout = _strip_ansi((getattr(result, "stdout", "") or "").strip())
|
||||
stderr = _strip_ansi((getattr(result, "stderr", "") or "").strip())
|
||||
combined = (
|
||||
(stdout + "\n" + stderr).strip() if (stdout and stderr) else (stdout or stderr)
|
||||
)
|
||||
return combined or _strip_ansi((result.output or "").strip())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("global options test env is clean")
|
||||
def step_global_opts_env_clean(context: Context) -> None:
|
||||
"""Ensure test-specific env vars are not polluted from previous tests.
|
||||
|
||||
Registers a cleanup handler that restores the original env var values after
|
||||
the scenario ends. The global ``after_scenario`` hook in
|
||||
``features/environment.py`` runs all ``context._cleanup_handlers``.
|
||||
"""
|
||||
orig_data_dir = os.environ.get("CLEVERAGENTS_DATA_DIR")
|
||||
orig_config_path = os.environ.get("CLEVERAGENTS_CONFIG_PATH")
|
||||
# Clean up any leftover from previous scenarios
|
||||
for var in ("CLEVERAGENTS_DATA_DIR", "CLEVERAGENTS_CONFIG_PATH"):
|
||||
os.environ.pop(var, None)
|
||||
|
||||
# Register restoration handler so that env vars set by main_callback()
|
||||
# during CLI invocation do not pollute subsequent scenarios.
|
||||
def _restore() -> None:
|
||||
for key, val in [
|
||||
("CLEVERAGENTS_DATA_DIR", orig_data_dir),
|
||||
("CLEVERAGENTS_CONFIG_PATH", orig_config_path),
|
||||
]:
|
||||
if val is not None:
|
||||
os.environ[key] = val
|
||||
else:
|
||||
os.environ.pop(key, None)
|
||||
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(_restore)
|
||||
|
||||
|
||||
@given("a temp data dir is prepared")
|
||||
def step_temp_data_dir_prepared(context: Context) -> None:
|
||||
"""Create a real temporary directory for --data-dir testing.
|
||||
|
||||
Registers a cleanup handler so the directory is removed after the scenario.
|
||||
"""
|
||||
tmp_dir = tempfile.mkdtemp(prefix="ca_test_data_dir_")
|
||||
context.temp_data_dir = tmp_dir
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
|
||||
|
||||
|
||||
@given("a temp config file is prepared")
|
||||
def step_temp_config_file_prepared(context: Context) -> None:
|
||||
"""Create a real temporary TOML config file for --config-path testing.
|
||||
|
||||
Registers a cleanup handler so the file is removed after the scenario.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".toml", prefix="ca_test_config_", delete=False
|
||||
) as tmp_file:
|
||||
tmp_file.write(b"# test config\n")
|
||||
tmp_path = tmp_file.name
|
||||
context.temp_config_file = tmp_path
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: os.unlink(tmp_path))
|
||||
|
||||
|
||||
@given("a temp file path is prepared")
|
||||
def step_temp_file_path_prepared(context: Context) -> None:
|
||||
"""Create a temporary FILE (not directory) to trigger --data-dir validation.
|
||||
|
||||
Registers a cleanup handler so the file is removed after the scenario.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(prefix="ca_test_file_", delete=False) as tmp_file:
|
||||
tmp_path = tmp_file.name
|
||||
context.temp_file_path = tmp_path
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: os.unlink(tmp_path))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps - CLI invocation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I run the global options CLI with data-dir flag and "{subcommand}"')
|
||||
def step_run_global_opts_data_dir(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with --data-dir pointing at the temp directory."""
|
||||
args = ["--data-dir", context.temp_data_dir, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with config-path flag and "{subcommand}"')
|
||||
def step_run_global_opts_config_path(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with --config-path pointing at the temp config file."""
|
||||
args = ["--config-path", context.temp_config_file, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with both path flags and "{subcommand}"')
|
||||
def step_run_global_opts_both(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with both --data-dir and --config-path."""
|
||||
args = [
|
||||
"--data-dir",
|
||||
context.temp_data_dir,
|
||||
"--config-path",
|
||||
context.temp_config_file,
|
||||
subcommand,
|
||||
]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with data-dir as temp file and "{subcommand}"')
|
||||
def step_run_global_opts_data_dir_file(context: Context, subcommand: str) -> None:
|
||||
"""Run CLI with --data-dir pointing to a FILE (triggers validation error)."""
|
||||
args = ["--data-dir", context.temp_file_path, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with invalid config-path and "{subcommand}"')
|
||||
def step_run_global_opts_invalid_config_path(context: Context, subcommand: str) -> None:
|
||||
"""Run CLI with --config-path pointing to a non-existent file."""
|
||||
args = ["--config-path", "/tmp/no_such_file_xyz_99999.toml", subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with args "{args}"')
|
||||
def step_run_global_opts_with_args(context: Context, args: str) -> None:
|
||||
"""Run the CLI with space-separated args string."""
|
||||
result = _invoke(args.split())
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps - direct callback testing for verbosity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call main_callback with verbosity {count:d}")
|
||||
def step_call_main_callback_verbosity(context: Context, count: int) -> None:
|
||||
"""Invoke main_callback directly, capturing the log level configured."""
|
||||
from cleveragents.cli.main import main_callback
|
||||
|
||||
context.global_opts_captured_log_level: str | None = None
|
||||
context.global_opts_captured_ctx_obj: dict[str, Any] = {}
|
||||
|
||||
def _capture_log_level(
|
||||
*, env: str = "development", log_level: str = "INFO"
|
||||
) -> None:
|
||||
context.global_opts_captured_log_level = log_level
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx.obj = {}
|
||||
|
||||
with patch(
|
||||
"cleveragents.config.logging.configure_structlog",
|
||||
side_effect=_capture_log_level,
|
||||
):
|
||||
main_callback(
|
||||
ctx=mock_ctx,
|
||||
version=None, # type: ignore[arg-type]
|
||||
show_secrets=False,
|
||||
fmt=OutputFormat.RICH,
|
||||
data_dir=None,
|
||||
config_path=None,
|
||||
verbose=count,
|
||||
)
|
||||
|
||||
context.global_opts_captured_ctx_obj = mock_ctx.obj
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - exit codes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the global options CLI should succeed")
|
||||
def step_global_opts_cli_succeed(context: Context) -> None:
|
||||
actual = context.global_opts_exit_code
|
||||
output = context.global_opts_output
|
||||
assert actual == 0, f"Expected exit code 0, got {actual}.\nOutput:\n{output}"
|
||||
|
||||
|
||||
@then("the global options CLI should fail")
|
||||
def step_global_opts_cli_fail(context: Context) -> None:
|
||||
actual = context.global_opts_exit_code
|
||||
assert actual != 0, (
|
||||
f"Expected non-zero exit code, got {actual}.\nOutput:\n{context.global_opts_output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - env var assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the CLEVERAGENTS_DATA_DIR env var should match the temp data dir")
|
||||
def step_env_data_dir_matches(context: Context) -> None:
|
||||
"""Verify that --data-dir set CLEVERAGENTS_DATA_DIR to the temp directory."""
|
||||
env_val = os.environ.get("CLEVERAGENTS_DATA_DIR")
|
||||
expected = str(Path(context.temp_data_dir).resolve())
|
||||
assert env_val is not None, (
|
||||
"CLEVERAGENTS_DATA_DIR was not set after --data-dir invocation"
|
||||
)
|
||||
assert str(Path(env_val).resolve()) == expected, (
|
||||
f"CLEVERAGENTS_DATA_DIR={env_val!r} does not match expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file")
|
||||
def step_env_config_path_matches(context: Context) -> None:
|
||||
"""Verify that --config-path set CLEVERAGENTS_CONFIG_PATH to the temp file."""
|
||||
env_val = os.environ.get("CLEVERAGENTS_CONFIG_PATH")
|
||||
expected = str(Path(context.temp_config_file).resolve())
|
||||
assert env_val is not None, (
|
||||
"CLEVERAGENTS_CONFIG_PATH was not set after --config-path invocation"
|
||||
)
|
||||
assert str(Path(env_val).resolve()) == expected, (
|
||||
f"CLEVERAGENTS_CONFIG_PATH={env_val!r} does not match expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - ctx.obj assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options ctx obj "{key}" should be {value:d}')
|
||||
def step_global_opts_ctx_obj_int(context: Context, key: str, value: int) -> None:
|
||||
"""Verify ctx.obj[key] has the expected integer value."""
|
||||
obj = context.global_opts_captured_ctx_obj
|
||||
assert key in obj, f"ctx.obj missing key '{key}'. ctx.obj={obj}"
|
||||
actual = obj[key]
|
||||
assert actual == value, f"ctx.obj['{key}']={actual!r}, expected {value!r}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - log level assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options log level should be "{level}"')
|
||||
def step_global_opts_log_level(context: Context, level: str) -> None:
|
||||
"""Verify that configure_structlog was called with the expected log level."""
|
||||
actual = context.global_opts_captured_log_level
|
||||
assert actual is not None, "configure_structlog was not called"
|
||||
assert actual.upper() == level.upper(), (
|
||||
f"Expected log level {level!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - output assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options output should contain "{text}"')
|
||||
def step_global_opts_output_contains(context: Context, text: str) -> None:
|
||||
output = context.global_opts_output
|
||||
assert text in output, f"Expected '{text}' in output:\n{output}"
|
||||
@@ -116,7 +116,10 @@ def step_impl(context):
|
||||
graph.execute = exec_with_messages
|
||||
|
||||
executor = bridge._create_graph_executor({"graph": "test_graph"})
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
|
||||
def _run_operator(message):
|
||||
results = []
|
||||
@@ -146,6 +149,10 @@ def step_impl(context):
|
||||
except ValueError:
|
||||
context.results.append("graph_executor_error")
|
||||
|
||||
# Close the fresh event loop to prevent resource leak
|
||||
with __import__("contextlib").suppress(RuntimeError):
|
||||
loop.close() # Safe: get_running_loop catches if another loop already exists
|
||||
|
||||
|
||||
@when("I exercise state utilities and node operator")
|
||||
def step_impl(context):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import tempfile
|
||||
@@ -29,7 +30,10 @@ def _write_config_file(data: dict[str, Any]) -> Path:
|
||||
|
||||
def _run_async(coro):
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
if loop.is_closed():
|
||||
raise RuntimeError
|
||||
if loop.is_running():
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
@@ -153,7 +154,10 @@ def _ensure_topological_levels(graph):
|
||||
|
||||
def _run_async(coro):
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
if loop.is_closed():
|
||||
raise RuntimeError
|
||||
if loop.is_running():
|
||||
@@ -435,7 +439,7 @@ def step_graph_route_idle_time(context):
|
||||
@given("I have a graph state that has been idle too long")
|
||||
def step_graph_state_idle(context):
|
||||
context.graph_state = GraphState()
|
||||
current_time = asyncio.get_event_loop().time()
|
||||
current_time = time.time()
|
||||
context.graph_state.metadata = {"last_updated": current_time - 10.0}
|
||||
context.graph_state.messages = []
|
||||
|
||||
@@ -673,7 +677,11 @@ def step_attempt_downgrade(context):
|
||||
)
|
||||
return
|
||||
try:
|
||||
context.downgrade_result = asyncio.get_event_loop().run_until_complete(
|
||||
try:
|
||||
_loop_downgrade_result = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_loop_downgrade_result = asyncio.new_event_loop()
|
||||
context.downgrade_result = _loop_downgrade_result.run_until_complete(
|
||||
context.route_bridge.downgrade_graph_to_stream(
|
||||
context.route_config, context.langgraph
|
||||
)
|
||||
@@ -709,7 +717,11 @@ def step_route_upgrade_downgrade(context):
|
||||
|
||||
@when("I perform an upgrade and then downgrade cycle")
|
||||
def step_cycle_upgrade_downgrade(context):
|
||||
context.langgraph = asyncio.get_event_loop().run_until_complete(
|
||||
try:
|
||||
_loop_langgraph = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_loop_langgraph = asyncio.new_event_loop()
|
||||
context.langgraph = _loop_langgraph.run_until_complete(
|
||||
context.route_bridge.upgrade_stream_to_graph(
|
||||
context.route_config, context.stream_message
|
||||
)
|
||||
@@ -731,7 +743,15 @@ def step_cycle_upgrade_downgrade(context):
|
||||
name=name, type=getattr(node, "type", NodeType.FUNCTION)
|
||||
)
|
||||
|
||||
context.stream_config = asyncio.get_event_loop().run_until_complete(
|
||||
try:
|
||||
|
||||
_loop_stream_config = asyncio.get_running_loop()
|
||||
|
||||
except RuntimeError:
|
||||
|
||||
_loop_stream_config = asyncio.new_event_loop()
|
||||
|
||||
context.stream_config = _loop_stream_config.run_until_complete(
|
||||
context.route_bridge.downgrade_graph_to_stream(
|
||||
RouteConfig.from_graph_config(
|
||||
GraphConfig(
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
"""Step definitions for TDD quality gate feature tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
# Ensure the project root is importable.
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from scripts.tdd_quality_gate import ( # noqa: E402
|
||||
check_expected_fail_removed,
|
||||
find_tdd_tests,
|
||||
parse_bug_refs,
|
||||
run_quality_gate,
|
||||
)
|
||||
from scripts.tdd_quality_gate import main as quality_gate_main # noqa: E402
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_temp_dir(context: object) -> Path:
|
||||
"""Return (and lazily create) the temporary directory on context."""
|
||||
tmp: Path | None = getattr(context, "temp_dir", None)
|
||||
if tmp is None:
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
context.temp_dir = tmp # type: ignore[attr-defined]
|
||||
return tmp
|
||||
|
||||
|
||||
def _cleanup_temp_dir(context: object) -> None:
|
||||
"""Remove the temporary directory if it exists."""
|
||||
tmp: Path | None = getattr(context, "temp_dir", None)
|
||||
if tmp is not None and tmp.exists():
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def _default_pr_diff_for_bug_refs(
|
||||
bug_refs: list[int], search_root: Path | None = None
|
||||
) -> str:
|
||||
"""Return a synthetic PR diff that removes expected-fail tags.
|
||||
|
||||
When *search_root* is provided the helper inspects the temp tree to
|
||||
decide whether each bug's TDD test lives in a ``.feature`` or
|
||||
``.robot`` file and emits the diff in the matching format.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
for bug_num in bug_refs:
|
||||
use_robot = False
|
||||
if search_root is not None:
|
||||
robot_hits = list(search_root.rglob("*.robot"))
|
||||
for rp in robot_hits:
|
||||
try:
|
||||
if f"tdd_bug_{bug_num}" in rp.read_text(encoding="utf-8"):
|
||||
use_robot = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if use_robot:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
f"diff --git a/robot/bug{bug_num}.robot b/robot/bug{bug_num}.robot",
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
else:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
f"diff --git a/features/bug{bug_num}.feature b/features/bug{bug_num}.feature",
|
||||
f"--- a/features/bug{bug_num}.feature",
|
||||
f"+++ b/features/bug{bug_num}.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}",
|
||||
f"+@tdd_bug @tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a PR description "{description}"')
|
||||
def step_given_pr_description(context: object, description: str) -> None:
|
||||
context.pr_description = description # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@given("a multiline PR description")
|
||||
def step_given_multiline_pr_description(context: object) -> None:
|
||||
context.pr_description = context.text # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I parse the bug references")
|
||||
def step_when_parse_bug_refs(context: object) -> None:
|
||||
context.bug_refs = parse_bug_refs( # type: ignore[attr-defined]
|
||||
context.pr_description # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then("the bug references should be [{refs}]")
|
||||
def step_then_bug_refs_should_be(context: object, refs: str) -> None:
|
||||
if refs.strip() == "":
|
||||
expected: list[int] = []
|
||||
else:
|
||||
expected = [int(x.strip()) for x in refs.split(",")]
|
||||
actual: list[int] = context.bug_refs # type: ignore[attr-defined]
|
||||
if actual != expected:
|
||||
raise AssertionError(f"Expected bug refs {expected}, got {actual}")
|
||||
|
||||
|
||||
@then("the bug references should be []")
|
||||
def step_then_bug_refs_empty(context: object) -> None:
|
||||
actual: list[int] = context.bug_refs # type: ignore[attr-defined]
|
||||
if actual != []:
|
||||
raise AssertionError(f"Expected empty bug refs, got {actual}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a temporary directory with a file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_dir_with_file(context: object, filepath: str, content: str) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given('a temporary directory also has a file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_dir_also_has_file(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@when("I search for TDD tests for bug {bug_num:d}")
|
||||
def step_when_search_tdd_tests(context: object, bug_num: int) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
context.found_tests = find_tdd_tests(bug_num, tmp) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("the search should find {count:d} test file")
|
||||
def step_then_search_finds_count(context: object, count: int) -> None:
|
||||
actual = len(context.found_tests) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(f"Expected {count} test file(s), found {actual}")
|
||||
|
||||
|
||||
@then("the search should find {count:d} test files")
|
||||
def step_then_search_finds_count_plural(context: object, count: int) -> None:
|
||||
actual = len(context.found_tests) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(f"Expected {count} test file(s), found {actual}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag removal verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I check expected fail removal for bug {bug_num:d}")
|
||||
def step_when_check_removal(context: object, bug_num: int) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
# Filter by bug tag first — matching the production path in run_quality_gate.
|
||||
test_files = find_tdd_tests(bug_num, tmp)
|
||||
context.removal_errors = check_expected_fail_removed( # type: ignore[attr-defined]
|
||||
test_files, bug_num
|
||||
)
|
||||
|
||||
|
||||
@then("there should be {count:d} removal error")
|
||||
def step_then_removal_errors_count(context: object, count: int) -> None:
|
||||
actual = len(context.removal_errors) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(
|
||||
f"Expected {count} removal error(s), got {actual}: {context.removal_errors}" # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then("there should be {count:d} removal errors")
|
||||
def step_then_removal_errors_count_plural(context: object, count: int) -> None:
|
||||
actual = len(context.removal_errors) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(
|
||||
f"Expected {count} removal error(s), got {actual}: {context.removal_errors}" # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then('the removal error should mention "{text}"')
|
||||
def step_then_removal_error_mentions(context: object, text: str) -> None:
|
||||
errors: list[str] = context.removal_errors # type: ignore[attr-defined]
|
||||
found = any(text in err for err in errors)
|
||||
if not found:
|
||||
raise AssertionError(
|
||||
f"Expected removal error mentioning '{text}', got: {errors}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full quality gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a temporary search root")
|
||||
def step_given_temp_search_root(context: object) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
_ensure_temp_dir(context)
|
||||
context.pr_diff = None # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@given('a temporary search root with file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_search_root_with_file(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
context.pr_diff = None # type: ignore[attr-defined]
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given('the search root also has file "{filepath}" containing "{content}"')
|
||||
def step_given_search_root_also_has(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given("the PR diff does not remove expected fail tags")
|
||||
def step_given_pr_diff_no_expected_fail_removal(context: object) -> None:
|
||||
context.pr_diff = "" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I run the quality gate")
|
||||
def step_when_run_quality_gate(context: object) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
pr_desc: str = getattr(context, "pr_description", "")
|
||||
pr_diff: str | None = getattr(context, "pr_diff", None)
|
||||
if pr_diff is None:
|
||||
bug_refs = parse_bug_refs(pr_desc)
|
||||
pr_diff = _default_pr_diff_for_bug_refs(bug_refs, tmp)
|
||||
errors, _bug_refs = run_quality_gate(pr_desc, tmp, pr_diff=pr_diff)
|
||||
context.gate_errors = errors # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("the quality gate should pass")
|
||||
def step_then_gate_passes(context: object) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
if errors:
|
||||
raise AssertionError(f"Quality gate should pass but got errors: {errors}")
|
||||
|
||||
|
||||
@then("the quality gate should fail")
|
||||
def step_then_gate_fails(context: object) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
if not errors:
|
||||
raise AssertionError("Quality gate should fail but passed with no errors")
|
||||
|
||||
|
||||
@then('the quality gate errors should mention "{text}"')
|
||||
def step_then_gate_errors_mention(context: object, text: str) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
found = any(text in err for err in errors)
|
||||
if not found:
|
||||
raise AssertionError(
|
||||
f"Expected quality gate error mentioning '{text}', got: {errors}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call parse_bug_refs with a non-string argument")
|
||||
def step_when_parse_bug_refs_non_string(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
parse_bug_refs(123) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with bug number {num:d}")
|
||||
def step_when_find_tdd_tests_invalid(context: object, num: int) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(num, Path("/tmp/nonexistent"))
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with a non-Path search root")
|
||||
def step_when_find_tdd_tests_non_path(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(1, "/tmp/nonexistent") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("a TypeError should be raised by the quality gate")
|
||||
def step_then_type_error_raised(context: object) -> None:
|
||||
exc = getattr(context, "caught_exception", None)
|
||||
if exc is None:
|
||||
raise AssertionError("Expected TypeError but no exception was raised")
|
||||
if not isinstance(exc, TypeError):
|
||||
raise AssertionError(f"Expected TypeError, got {type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
@then("a ValueError should be raised by the quality gate")
|
||||
def step_then_value_error_raised(context: object) -> None:
|
||||
exc = getattr(context, "caught_exception", None)
|
||||
if exc is None:
|
||||
raise AssertionError("Expected ValueError but no exception was raised")
|
||||
if not isinstance(exc, ValueError):
|
||||
raise AssertionError(f"Expected ValueError, got {type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Robot-format diff, unreadable files, and extra argument validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the PR diff removes expected fail for robot bug {bug_num:d}")
|
||||
def step_given_robot_diff_removes_expected_fail(context: object, bug_num: int) -> None:
|
||||
context.pr_diff = "\n".join( # type: ignore[attr-defined]
|
||||
[
|
||||
f"diff --git a/robot/bug{bug_num}.robot b/robot/bug{bug_num}.robot",
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@given("a temporary directory with an unreadable feature file for bug {bug_num:d}")
|
||||
def step_given_unreadable_feature_file(context: object, bug_num: int) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / "features" / "bug.feature"
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Write invalid UTF-8 bytes so read_text(encoding="utf-8") raises
|
||||
# UnicodeDecodeError (caught as OSError subclass). This is root-safe
|
||||
# unlike chmod(0o000) which root bypasses.
|
||||
full_path.write_bytes(
|
||||
f"@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}".encode() + b"\xff\xfe"
|
||||
)
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with a non-list argument")
|
||||
def step_when_check_expected_fail_removed_non_list(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed("not-a-list", 1) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with bug number {num:d}")
|
||||
def step_when_check_expected_fail_removed_bad_bug(context: object, num: int) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed([], num)
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool type guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with boolean True as bug number")
|
||||
def step_when_find_tdd_tests_bool(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(True, Path("/tmp/nonexistent")) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with boolean True as bug number")
|
||||
def step_when_check_expected_fail_removed_bool(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed([], True) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Co-located bug false positive guard (M1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the PR diff only removes expected fail for bug {other:d} not bug {target:d}")
|
||||
def step_given_pr_diff_removes_wrong_bug(
|
||||
context: object, other: int, target: int
|
||||
) -> None:
|
||||
# Craft a diff that removes expected-fail for bug `other` but NOT for
|
||||
# bug `target`. The diff must mention bug `other`'s tag on the removed
|
||||
# line, so the gate should NOT count this as a removal for `target`.
|
||||
context.pr_diff = "\n".join( # type: ignore[attr-defined]
|
||||
[
|
||||
"diff --git a/features/bugs99.feature b/features/bugs99.feature",
|
||||
"--- a/features/bugs99.feature",
|
||||
"+++ b/features/bugs99.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{other}",
|
||||
f"+@tdd_bug @tdd_bug_{other}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_quality_gate argument validation (L4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-string PR description")
|
||||
def step_when_run_quality_gate_non_str_desc(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate(123, Path("/tmp")) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-Path search root")
|
||||
def step_when_run_quality_gate_non_path_root(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", "/tmp") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with an empty base_ref")
|
||||
def step_when_run_quality_gate_empty_base_ref(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", Path("/tmp"), base_ref=" ")
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-string pr_diff")
|
||||
def step_when_run_quality_gate_non_str_diff(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", Path("/tmp"), pr_diff=123) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# main() CLI entry point (M2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I call main with PR_DESCRIPTION "{description}"')
|
||||
def step_when_call_main_with_desc(context: object, description: str) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
saved_desc = os.environ.get("PR_DESCRIPTION")
|
||||
saved_cwd = os.getcwd()
|
||||
try:
|
||||
os.environ["PR_DESCRIPTION"] = description
|
||||
os.chdir(tmp)
|
||||
context.main_exit_code = quality_gate_main() # type: ignore[attr-defined]
|
||||
finally:
|
||||
os.chdir(saved_cwd)
|
||||
if saved_desc is None:
|
||||
os.environ.pop("PR_DESCRIPTION", None)
|
||||
else:
|
||||
os.environ["PR_DESCRIPTION"] = saved_desc
|
||||
|
||||
|
||||
@when('I call main with PR_DESCRIPTION "{description}" in an empty search root')
|
||||
def step_when_call_main_missing_test(context: object, description: str) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
saved_desc = os.environ.get("PR_DESCRIPTION")
|
||||
saved_base = os.environ.get("PR_BASE_REF")
|
||||
saved_cwd = os.getcwd()
|
||||
try:
|
||||
os.environ["PR_DESCRIPTION"] = description
|
||||
# Use a base_ref that won't exist in the temp dir (not a git repo),
|
||||
# so _collect_pr_diff will fail and produce an error about the diff.
|
||||
os.environ["PR_BASE_REF"] = "master"
|
||||
os.chdir(tmp)
|
||||
context.main_exit_code = quality_gate_main() # type: ignore[attr-defined]
|
||||
finally:
|
||||
os.chdir(saved_cwd)
|
||||
if saved_desc is None:
|
||||
os.environ.pop("PR_DESCRIPTION", None)
|
||||
else:
|
||||
os.environ["PR_DESCRIPTION"] = saved_desc
|
||||
if saved_base is None:
|
||||
os.environ.pop("PR_BASE_REF", None)
|
||||
else:
|
||||
os.environ["PR_BASE_REF"] = saved_base
|
||||
|
||||
|
||||
@then("the main exit code should be {code:d}")
|
||||
def step_then_main_exit_code(context: object, code: int) -> None:
|
||||
actual = context.main_exit_code # type: ignore[attr-defined]
|
||||
if actual != code:
|
||||
raise AssertionError(f"Expected main exit code {code}, got {actual}")
|
||||
@@ -0,0 +1,265 @@
|
||||
Feature: TDD bug tag quality gate for bug fix PRs
|
||||
As a CI system
|
||||
I want to enforce TDD bug fix workflow rules on PRs
|
||||
So that bug fix PRs follow the required TDD workflow
|
||||
|
||||
# --- PR description parsing ---
|
||||
|
||||
Scenario: Parse single Fixes reference
|
||||
Given a PR description "Fixes #42"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
Scenario: Parse single Closes reference
|
||||
Given a PR description "Closes #100"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [100]
|
||||
|
||||
Scenario: Parse single Resolves reference
|
||||
Given a PR description "Resolves #7"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [7]
|
||||
|
||||
Scenario: Parse case-insensitive closing keywords
|
||||
Given a PR description "fixes #10 and CLOSES #20"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [10, 20]
|
||||
|
||||
Scenario: Parse ISSUES CLOSED block
|
||||
Given a PR description "ISSUES CLOSED: #5, #10"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [5, 10]
|
||||
|
||||
Scenario: Parse multiple mixed references
|
||||
Given a PR description "Fixes #42, also closes #99. ISSUES CLOSED: #7"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [7, 42, 99]
|
||||
|
||||
Scenario: No bug references returns empty list
|
||||
Given a PR description "Add new feature for users"
|
||||
When I parse the bug references
|
||||
Then the bug references should be []
|
||||
|
||||
Scenario: Deduplicate repeated bug references
|
||||
Given a PR description "Fixes #42. Also closes #42"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
Scenario: Parse past tense closing keywords
|
||||
Given a PR description "Fixed #15, Closed #20, Resolved #25"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [15, 20, 25]
|
||||
|
||||
Scenario: Ignore non-closing words that end with keyword substrings
|
||||
Given a PR description "prefixes #12 and hotfixes #34"
|
||||
When I parse the bug references
|
||||
Then the bug references should be []
|
||||
|
||||
Scenario: Parse bug reference in multi-line PR description
|
||||
Given a multiline PR description
|
||||
"""
|
||||
feat(ci): implement quality gate
|
||||
|
||||
This PR adds the TDD quality gate.
|
||||
|
||||
Fixes #42
|
||||
"""
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
# --- TDD test search ---
|
||||
|
||||
Scenario: Find TDD test in .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 1 test file
|
||||
|
||||
Scenario: Find TDD test in .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 1 test file
|
||||
|
||||
Scenario: Find TDD tests in both .feature and .robot files
|
||||
Given a temporary directory with a file "features/bug.feature" containing "@tdd_bug_42"
|
||||
And a temporary directory also has a file "robot/bug.robot" containing "tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 2 test files
|
||||
|
||||
Scenario: No TDD test found for bug number
|
||||
Given a temporary directory with a file "tests/other.feature" containing "@tdd_bug_99"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
Scenario: Do not match partial TDD bug tags
|
||||
Given a temporary directory with a file "tests/partial.feature" containing "@tdd_bug_420"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
# --- Tag removal verification ---
|
||||
|
||||
Scenario: Expected fail tag still present in .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 1 removal error
|
||||
And the removal error should mention "@tdd_expected_fail"
|
||||
And the removal error should mention "@tdd_bug_42"
|
||||
|
||||
Scenario: Expected fail tag removed from .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
Scenario: Expected fail tag still present in .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_expected_fail tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 1 removal error
|
||||
|
||||
Scenario: Expected fail tag removed from .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_bug tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
# --- Full quality gate ---
|
||||
|
||||
Scenario: Quality gate passes when no bug refs in PR
|
||||
Given a temporary search root
|
||||
And a PR description "Add new feature"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate fails when no TDD test exists for referenced bug
|
||||
Given a temporary search root
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No TDD test found for bug #42"
|
||||
|
||||
Scenario: Quality gate fails when expected fail tag is still present
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "@tdd_expected_fail"
|
||||
|
||||
Scenario: Quality gate passes when expected fail tag has been removed
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate handles multiple bug references
|
||||
Given a temporary search root with file "features/bug10.feature" containing "@tdd_bug @tdd_bug_10"
|
||||
And the search root also has file "features/bug20.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_20"
|
||||
And a PR description "Fixes #10 and fixes #20"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "@tdd_bug_20"
|
||||
|
||||
Scenario: Quality gate passes when all bugs have clean TDD tests
|
||||
Given a temporary search root with file "features/bug10.feature" containing "@tdd_bug @tdd_bug_10"
|
||||
And the search root also has file "robot/bug20.robot" containing "tdd_bug tdd_bug_20"
|
||||
And a PR description "Fixes #10 and fixes #20"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate fails when PR diff does not remove expected fail tags
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff does not remove expected fail tags
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No removal of @tdd_expected_fail / tdd_expected_fail detected"
|
||||
|
||||
Scenario: Quality gate passes for robot diff with expected fail removed across hunks
|
||||
Given a temporary search root with file "robot/bug.robot" containing "tdd_bug tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff removes expected fail for robot bug 42
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Issue number zero is silently ignored
|
||||
Given a temporary search root
|
||||
And a PR description "Fixes #0"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: find_tdd_tests skips unreadable files
|
||||
Given a temporary directory with an unreadable feature file for bug 42
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
Scenario: check_expected_fail_removed skips unreadable files
|
||||
Given a temporary directory with an unreadable feature file for bug 42
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
# --- Argument validation ---
|
||||
|
||||
Scenario: parse_bug_refs rejects non-string input
|
||||
When I call parse_bug_refs with a non-string argument
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: find_tdd_tests rejects invalid bug number
|
||||
When I call find_tdd_tests with bug number 0
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: find_tdd_tests rejects non-Path search root
|
||||
When I call find_tdd_tests with a non-Path search root
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects non-list input
|
||||
When I call check_expected_fail_removed with a non-list argument
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects invalid bug number
|
||||
When I call check_expected_fail_removed with bug number 0
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
# --- Bool type guard ---
|
||||
|
||||
Scenario: find_tdd_tests rejects boolean True as bug number
|
||||
When I call find_tdd_tests with boolean True as bug number
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects boolean True as bug number
|
||||
When I call check_expected_fail_removed with boolean True as bug number
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
# --- Co-located bug false positive guard (M1) ---
|
||||
|
||||
Scenario: Diff detection does not false-positive for co-located bug tests
|
||||
Given a temporary search root with file "features/bugs.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And the search root also has file "features/bugs99.feature" containing "@tdd_bug @tdd_bug_99"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff only removes expected fail for bug 99 not bug 42
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No removal of @tdd_expected_fail / tdd_expected_fail detected"
|
||||
|
||||
# --- run_quality_gate argument validation (L4) ---
|
||||
|
||||
Scenario: run_quality_gate rejects non-string PR description
|
||||
When I call run_quality_gate with a non-string PR description
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects non-Path search root
|
||||
When I call run_quality_gate with a non-Path search root
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects empty base_ref
|
||||
When I call run_quality_gate with an empty base_ref
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects non-string pr_diff
|
||||
When I call run_quality_gate with a non-string pr_diff
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
# --- main() CLI entry point (M2) ---
|
||||
|
||||
Scenario: main returns 0 when no bug refs in PR description
|
||||
When I call main with PR_DESCRIPTION "Add new feature"
|
||||
Then the main exit code should be 0
|
||||
|
||||
Scenario: main returns 1 when TDD test is missing
|
||||
When I call main with PR_DESCRIPTION "Fixes #99999" in an empty search root
|
||||
Then the main exit code should be 1
|
||||
+23
@@ -882,6 +882,29 @@ def benchmark_regression(session: nox.Session):
|
||||
session.run("asv", "publish", f"--config={config_path}")
|
||||
|
||||
|
||||
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
||||
def tdd_quality_gate(session: nox.Session):
|
||||
"""Enforce TDD bug fix workflow rules on PRs.
|
||||
|
||||
Reads the PR description from the ``PR_DESCRIPTION`` environment
|
||||
variable and verifies that:
|
||||
|
||||
1. Every bug referenced via closing keywords (``Fixes #N``,
|
||||
``Closes #N``, ``Resolves #N``, ``ISSUES CLOSED: #N``) has
|
||||
a corresponding TDD test tagged ``@tdd_bug_N``.
|
||||
2. The ``@tdd_expected_fail`` / ``tdd_expected_fail`` tag has been
|
||||
removed from each of those tests in the PR diff.
|
||||
|
||||
If no bug references are found the gate passes trivially.
|
||||
"""
|
||||
# The quality gate script uses only standard library; no install needed.
|
||||
pr_description = os.environ.get("PR_DESCRIPTION", "")
|
||||
pr_base_ref = os.environ.get("PR_BASE_REF", "master")
|
||||
session.env["PR_DESCRIPTION"] = pr_description
|
||||
session.env["PR_BASE_REF"] = pr_base_ref
|
||||
session.run("python", "scripts/tdd_quality_gate.py")
|
||||
|
||||
|
||||
# Sessions to run by default when running `nox` without arguments
|
||||
nox.options.sessions = [
|
||||
"lint", # ~5-10 seconds
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for CLI global options --data-dir, --config-path, and -v.
|
||||
...
|
||||
... Verifies that the spec-required global options are properly accepted
|
||||
... and propagated end-to-end (issue #6785).
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_cli_global_options.py
|
||||
|
||||
*** Test Cases ***
|
||||
Data Dir Option Accepted With Valid Directory
|
||||
[Documentation] Verify --data-dir is accepted without error for an existing directory.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} data-dir-accepted cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-data-dir-accepted-ok
|
||||
|
||||
Config Path Option Accepted With Valid File
|
||||
[Documentation] Verify --config-path is accepted without error for an existing file.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} config-path-accepted cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-config-path-accepted-ok
|
||||
|
||||
Verbosity Flag Accepted
|
||||
[Documentation] Verify -v flag is accepted without crashing.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} verbosity-accepted cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-verbosity-accepted-ok
|
||||
|
||||
Data Dir And Config Path Combined
|
||||
[Documentation] Verify --data-dir and --config-path can be combined end-to-end.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} combined-options cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-combined-ok
|
||||
|
||||
Data Dir Overrides Settings Data Dir
|
||||
[Documentation] Verify --data-dir properly overrides CLEVERAGENTS_DATA_DIR.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} data-dir-override cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-data-dir-override-ok
|
||||
|
||||
Config Path Overrides Config Service Path
|
||||
[Documentation] Verify --config-path properly overrides the config service path.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} config-path-override cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-config-path-override-ok
|
||||
|
||||
Verbosity Sets Log Level Correctly
|
||||
[Documentation] Verify -v count correctly maps to log levels.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} verbosity-levels cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-global-options-verbosity-levels-ok
|
||||
|
||||
Help Shows All Three Options
|
||||
[Documentation] Verify --help output includes --data-dir, --config-path, and -v.
|
||||
${result}= Run Process ${PYTHON} -m cleveragents --help
|
||||
... timeout=60s on_timeout=kill
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} --data-dir
|
||||
Should Contain ${result.stdout} --config-path
|
||||
Should Contain ${result.stdout} -v
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Helper script for cli_global_options.robot integration tests.
|
||||
|
||||
Each sub-command verifies one end-to-end behaviour of the new global CLI
|
||||
options (--data-dir, --config-path, -v) introduced by issue #6785.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _invoke(args: list[str]) -> tuple[int, str]:
|
||||
"""Invoke the Typer app and return (exit_code, combined_output)."""
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.main import app
|
||||
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(app, args, catch_exceptions=True)
|
||||
output = result.output or ""
|
||||
return result.exit_code, output
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test cases (sub-commands)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_data_dir_accepted() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
code, output = _invoke(["--data-dir", tmp, "version"])
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
print("cli-global-options-data-dir-accepted-ok")
|
||||
|
||||
|
||||
def test_config_path_accepted() -> None:
|
||||
with tempfile.NamedTemporaryFile(suffix=".toml", delete=False) as tmp:
|
||||
tmp.write(b"# test config\n")
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
code, output = _invoke(["--config-path", tmp_path, "version"])
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
print("cli-global-options-config-path-accepted-ok")
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
def test_verbosity_accepted() -> None:
|
||||
code, output = _invoke(["-v", "version"])
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
print("cli-global-options-verbosity-accepted-ok")
|
||||
|
||||
|
||||
def test_combined_options() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
with tempfile.NamedTemporaryFile(suffix=".toml", delete=False) as tmp_cfg:
|
||||
tmp_cfg.write(b"# test\n")
|
||||
tmp_cfg_path = tmp_cfg.name
|
||||
try:
|
||||
code, output = _invoke(
|
||||
[
|
||||
"--data-dir",
|
||||
tmp_dir,
|
||||
"--config-path",
|
||||
tmp_cfg_path,
|
||||
"version",
|
||||
]
|
||||
)
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
print("cli-global-options-combined-ok")
|
||||
finally:
|
||||
os.unlink(tmp_cfg_path)
|
||||
|
||||
|
||||
def test_data_dir_override() -> None:
|
||||
"""Verify --data-dir sets CLEVERAGENTS_DATA_DIR env var during invocation."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
code, output = _invoke(["--data-dir", tmp, "version"])
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
|
||||
# Verify the env var was set to (or derived from) the tmp dir
|
||||
env_val = os.environ.get("CLEVERAGENTS_DATA_DIR")
|
||||
if env_val:
|
||||
resolved_env = str(Path(env_val).resolve())
|
||||
resolved_tmp = str(Path(tmp).resolve())
|
||||
assert resolved_env == resolved_tmp, (
|
||||
f"CLEVERAGENTS_DATA_DIR={env_val!r} != {tmp!r}"
|
||||
)
|
||||
|
||||
print("cli-global-options-data-dir-override-ok")
|
||||
|
||||
|
||||
def test_config_path_override() -> None:
|
||||
"""Verify --config-path sets CLEVERAGENTS_CONFIG_PATH env var during invocation."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".toml", delete=False) as tmp:
|
||||
tmp.write(b"# test\n")
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
code, output = _invoke(["--config-path", tmp_path, "version"])
|
||||
assert code == 0, f"Expected exit 0, got {code}. Output: {output}"
|
||||
env_val = os.environ.get("CLEVERAGENTS_CONFIG_PATH")
|
||||
if env_val:
|
||||
assert str(Path(env_val).resolve()) == str(Path(tmp_path).resolve()), (
|
||||
f"CLEVERAGENTS_CONFIG_PATH={env_val!r} != {tmp_path!r}"
|
||||
)
|
||||
print("cli-global-options-config-path-override-ok")
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
def test_verbosity_levels() -> None:
|
||||
"""Verify the verbosity count → log level mapping."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from cleveragents.cli.formatting import OutputFormat
|
||||
from cleveragents.cli.main import main_callback
|
||||
|
||||
expected_levels = {
|
||||
0: "CRITICAL",
|
||||
1: "ERROR",
|
||||
2: "WARNING",
|
||||
3: "INFO",
|
||||
4: "DEBUG",
|
||||
5: "DEBUG",
|
||||
}
|
||||
|
||||
for count, expected in expected_levels.items():
|
||||
captured_levels: list[str] = []
|
||||
|
||||
def _make_capture(levels: list[str]): # type: ignore[no-untyped-def]
|
||||
def _capture(*, env: str = "development", log_level: str = "INFO") -> None:
|
||||
levels.append(log_level)
|
||||
|
||||
return _capture
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx.obj = {}
|
||||
|
||||
with patch(
|
||||
"cleveragents.config.logging.configure_structlog",
|
||||
side_effect=_make_capture(captured_levels),
|
||||
):
|
||||
main_callback(
|
||||
ctx=mock_ctx,
|
||||
version=None, # type: ignore[arg-type]
|
||||
show_secrets=False,
|
||||
fmt=OutputFormat.RICH,
|
||||
data_dir=None,
|
||||
config_path=None,
|
||||
verbose=count,
|
||||
)
|
||||
|
||||
assert captured_levels, f"configure_structlog not called for count={count}"
|
||||
actual = captured_levels[-1].upper()
|
||||
assert actual == expected, (
|
||||
f"verbose={count}: expected log_level={expected!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
print("cli-global-options-verbosity-levels-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"data-dir-accepted": test_data_dir_accepted,
|
||||
"config-path-accepted": test_config_path_accepted,
|
||||
"verbosity-accepted": test_verbosity_accepted,
|
||||
"combined-options": test_combined_options,
|
||||
"data-dir-override": test_data_dir_override,
|
||||
"config-path-override": test_config_path_override,
|
||||
"verbosity-levels": test_verbosity_levels,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: helper_cli_global_options.py <command>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
cmd = sys.argv[1]
|
||||
fn = _COMMANDS.get(cmd)
|
||||
if fn is None:
|
||||
print(f"Unknown command: {cmd}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
fn() # type: ignore[call-arg]
|
||||
except Exception as exc:
|
||||
print(f"FAILED: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,412 @@
|
||||
"""Helper for ``tdd_quality_gate.robot`` — exercises TDD quality gate logic.
|
||||
|
||||
Each sub-command exercises a specific aspect of the quality gate and
|
||||
prints a sentinel string on success. Exit 0 = check passed,
|
||||
1 = unexpected outcome.
|
||||
|
||||
See CONTRIBUTING.md > Bug Fix Workflow for the full specification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure the project root is importable.
|
||||
_ROOT = str(Path(__file__).resolve().parents[1])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from scripts.tdd_quality_gate import ( # noqa: E402
|
||||
find_tdd_tests,
|
||||
parse_bug_refs,
|
||||
run_quality_gate,
|
||||
)
|
||||
|
||||
|
||||
def _make_temp_tree(files: dict[str, str]) -> Path:
|
||||
"""Create a temporary directory with the given file tree."""
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
for filepath, content in files.items():
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
return tmp
|
||||
|
||||
|
||||
def _default_pr_diff_for_bug_refs(
|
||||
bug_refs: list[int], search_root: Path | None = None
|
||||
) -> str:
|
||||
"""Return a synthetic PR diff that removes expected-fail tags.
|
||||
|
||||
When *search_root* is provided the helper inspects the temp tree to
|
||||
decide whether each bug's TDD test lives in a ``.feature`` or
|
||||
``.robot`` file and emits the diff in the matching format.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
for bug_num in bug_refs:
|
||||
use_robot = False
|
||||
if search_root is not None:
|
||||
robot_hits = list(search_root.rglob("*.robot"))
|
||||
for rp in robot_hits:
|
||||
try:
|
||||
if f"tdd_bug_{bug_num}" in rp.read_text(encoding="utf-8"):
|
||||
use_robot = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if use_robot:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
(
|
||||
f"diff --git a/robot/bug{bug_num}.robot "
|
||||
f"b/robot/bug{bug_num}.robot"
|
||||
),
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
else:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
(
|
||||
f"diff --git a/features/bug{bug_num}.feature "
|
||||
f"b/features/bug{bug_num}.feature"
|
||||
),
|
||||
f"--- a/features/bug{bug_num}.feature",
|
||||
f"+++ b/features/bug{bug_num}.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}",
|
||||
f"+@tdd_bug @tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_fixes_single() -> int:
|
||||
"""Verify parsing a single Fixes #N reference."""
|
||||
refs = parse_bug_refs("Fixes #42")
|
||||
if refs != [42]:
|
||||
print(f"FAIL: expected [42], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-fixes-single-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_mixed_refs() -> int:
|
||||
"""Verify parsing multiple mixed closing keywords."""
|
||||
refs = parse_bug_refs("Fixes #42, also closes #99. Resolves #7")
|
||||
if refs != [7, 42, 99]:
|
||||
print(f"FAIL: expected [7, 42, 99], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-mixed-refs-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_issues_closed() -> int:
|
||||
"""Verify parsing ISSUES CLOSED: block."""
|
||||
refs = parse_bug_refs("ISSUES CLOSED: #5, #10")
|
||||
if refs != [5, 10]:
|
||||
print(f"FAIL: expected [5, 10], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-issues-closed-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_non_closing_words() -> int:
|
||||
"""Verify parser ignores embedded keyword substrings."""
|
||||
refs = parse_bug_refs("prefixes #12 and hotfixes #34")
|
||||
if refs != []:
|
||||
print(f"FAIL: expected [], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-non-closing-words-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def no_bug_refs_pass() -> int:
|
||||
"""Verify the quality gate passes when no bug refs in PR."""
|
||||
tmp = _make_temp_tree({})
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Add new feature", tmp)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("no-bug-refs-pass-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_feature_test() -> int:
|
||||
"""Verify finding @tdd_bug_N in .feature files."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 1:
|
||||
print(f"FAIL: expected 1 test, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-feature-test-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def find_robot_test() -> int:
|
||||
"""Verify finding tdd_bug_N in .robot files."""
|
||||
tmp = _make_temp_tree({"robot/bug.robot": "[Tags] tdd_bug tdd_bug_42\n"})
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 1:
|
||||
print(f"FAIL: expected 1 test, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-robot-test-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def find_exact_tag_match() -> int:
|
||||
"""Verify partial tags are not treated as exact bug tag matches."""
|
||||
tmp = _make_temp_tree({"features/partial.feature": "@tdd_bug_420\nFeature: Test\n"})
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if tests:
|
||||
print(f"FAIL: expected 0 tests, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-exact-tag-match-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality gate integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def no_tdd_test_fails() -> int:
|
||||
"""Verify the gate fails when no TDD test exists for a referenced bug."""
|
||||
tmp = _make_temp_tree({})
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff="")
|
||||
if not errors:
|
||||
print("FAIL: expected errors for missing TDD test", file=sys.stderr)
|
||||
return 1
|
||||
if not any("No TDD test found for bug #42" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected 'No TDD test found' error, got: {errors}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("no-tdd-test-fails-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def expected_fail_present() -> int:
|
||||
"""Verify the gate fails when @tdd_expected_fail is still present."""
|
||||
ef_content = "@tdd_expected_fail @tdd_bug @tdd_bug_42\nFeature: Test\n"
|
||||
tmp = _make_temp_tree({"features/bug.feature": ef_content})
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if not errors:
|
||||
print(
|
||||
"FAIL: expected errors for @tdd_expected_fail present", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
if not any("@tdd_expected_fail" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected '@tdd_expected_fail' error, got: {errors}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("expected-fail-present-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def expected_fail_removed() -> int:
|
||||
"""Verify the gate passes when @tdd_expected_fail has been removed."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("expected-fail-removed-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def multi_bug_mixed() -> int:
|
||||
"""Verify the gate handles multiple bugs with mixed outcomes."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug10.feature": "@tdd_bug @tdd_bug_10\nFeature: Bug 10\n",
|
||||
"features/bug20.feature": (
|
||||
"@tdd_expected_fail @tdd_bug @tdd_bug_20\nFeature: Bug 20\n"
|
||||
),
|
||||
}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([10, 20], tmp)
|
||||
errors, _refs = run_quality_gate(
|
||||
"Fixes #10 and fixes #20", tmp, pr_diff=pr_diff
|
||||
)
|
||||
if not errors:
|
||||
print("FAIL: expected errors for bug #20", file=sys.stderr)
|
||||
return 1
|
||||
if not any("@tdd_bug_20" in e for e in errors):
|
||||
print(f"FAIL: expected error about bug #20, got: {errors}", file=sys.stderr)
|
||||
return 1
|
||||
# Bug #10 should not have errors
|
||||
if any("@tdd_bug_10" in e for e in errors):
|
||||
print(f"FAIL: unexpected error about bug #10: {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("multi-bug-mixed-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def all_clean_passes() -> int:
|
||||
"""Verify the gate passes when all bugs have clean TDD tests."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug10.feature": "@tdd_bug @tdd_bug_10\nFeature: Bug 10\n",
|
||||
"robot/bug20.robot": "[Tags] tdd_bug tdd_bug_20\n",
|
||||
}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([10, 20], tmp)
|
||||
errors, _refs = run_quality_gate(
|
||||
"Fixes #10 and fixes #20", tmp, pr_diff=pr_diff
|
||||
)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("all-clean-passes-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def both_behave_and_robot() -> int:
|
||||
"""Verify the gate checks tests in both .feature and .robot files."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Bug\n",
|
||||
"robot/bug.robot": "[Tags] tdd_bug tdd_bug_42\n",
|
||||
}
|
||||
)
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 2:
|
||||
print(f"FAIL: expected 2 tests, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("both-behave-and-robot-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def diff_removal_required() -> int:
|
||||
"""Verify the gate fails when PR diff has no expected-fail removal."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff="")
|
||||
if not errors:
|
||||
print("FAIL: expected diff-removal error", file=sys.stderr)
|
||||
return 1
|
||||
if not any("No removal of @tdd_expected_fail" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected diff-removal message, got: {errors}", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
print("diff-removal-required-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, Callable[[], int]] = {
|
||||
"parse_fixes_single": parse_fixes_single,
|
||||
"parse_mixed_refs": parse_mixed_refs,
|
||||
"parse_issues_closed": parse_issues_closed,
|
||||
"parse_non_closing_words": parse_non_closing_words,
|
||||
"no_bug_refs_pass": no_bug_refs_pass,
|
||||
"find_feature_test": find_feature_test,
|
||||
"find_robot_test": find_robot_test,
|
||||
"find_exact_tag_match": find_exact_tag_match,
|
||||
"no_tdd_test_fails": no_tdd_test_fails,
|
||||
"expected_fail_present": expected_fail_present,
|
||||
"expected_fail_removed": expected_fail_removed,
|
||||
"multi_bug_mixed": multi_bug_mixed,
|
||||
"all_clean_passes": all_clean_passes,
|
||||
"both_behave_and_robot": both_behave_and_robot,
|
||||
"diff_removal_required": diff_removal_required,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Dispatch to the sub-command named in sys.argv[1]."""
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <command>", file=sys.stderr)
|
||||
print(f"Commands: {sorted(_COMMANDS)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
cmd = sys.argv[1]
|
||||
handler = _COMMANDS.get(cmd)
|
||||
if handler is None:
|
||||
print(f"Unknown command: {cmd}", file=sys.stderr)
|
||||
print(f"Available: {sorted(_COMMANDS)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
return handler()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,149 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for the TDD quality gate script.
|
||||
...
|
||||
... Exercises ``scripts/tdd_quality_gate.py`` end-to-end by
|
||||
... invoking it as a subprocess with various PR_DESCRIPTION
|
||||
... values and temporary file trees, then verifying exit codes
|
||||
... and output messages.
|
||||
...
|
||||
... See CONTRIBUTING.md > Bug Fix Workflow for the full
|
||||
... specification.
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_tdd_quality_gate.py
|
||||
|
||||
*** Test Cases ***
|
||||
# ===========================================================================
|
||||
# PR description parsing
|
||||
# ===========================================================================
|
||||
|
||||
Parse Single Fixes Reference
|
||||
[Documentation] Verify the script extracts a single Fixes #N reference
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_fixes_single
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-fixes-single-ok
|
||||
|
||||
Parse Multiple Mixed References
|
||||
[Documentation] Verify the script extracts multiple closing keywords
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_mixed_refs
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-mixed-refs-ok
|
||||
|
||||
Parse Issues Closed Block
|
||||
[Documentation] Verify the script extracts ISSUES CLOSED: #N references
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_issues_closed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-issues-closed-ok
|
||||
|
||||
Parse Non Closing Words
|
||||
[Documentation] Verify parser ignores embedded keyword substrings
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_non_closing_words
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-non-closing-words-ok
|
||||
|
||||
No Bug References Passes Trivially
|
||||
[Documentation] Verify the quality gate passes when no bug refs in PR
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} no_bug_refs_pass
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} no-bug-refs-pass-ok
|
||||
|
||||
# ===========================================================================
|
||||
# TDD test search and tag verification
|
||||
# ===========================================================================
|
||||
|
||||
Find TDD Test In Feature File
|
||||
[Documentation] Verify the script finds @tdd_bug_N in .feature files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_feature_test
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-feature-test-ok
|
||||
|
||||
Find TDD Test In Robot File
|
||||
[Documentation] Verify the script finds tdd_bug_N in .robot files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_robot_test
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-robot-test-ok
|
||||
|
||||
Find TDD Test Uses Exact Tag Match
|
||||
[Documentation] Verify partial tags are not matched as exact bug tags
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_exact_tag_match
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-exact-tag-match-ok
|
||||
|
||||
No TDD Test Found Fails Gate
|
||||
[Documentation] Verify the gate fails when no TDD test exists
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} no_tdd_test_fails
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} no-tdd-test-fails-ok
|
||||
|
||||
Expected Fail Tag Still Present Fails Gate
|
||||
[Documentation] Verify the gate fails when @tdd_expected_fail is still present
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} expected_fail_present
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} expected-fail-present-ok
|
||||
|
||||
Expected Fail Tag Removed Passes Gate
|
||||
[Documentation] Verify the gate passes when @tdd_expected_fail is removed
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} expected_fail_removed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} expected-fail-removed-ok
|
||||
|
||||
# ===========================================================================
|
||||
# Full quality gate integration
|
||||
# ===========================================================================
|
||||
|
||||
Full Gate Multiple Bugs Mixed Results
|
||||
[Documentation] Verify the gate handles multiple bugs with mixed outcomes
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} multi_bug_mixed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} multi-bug-mixed-ok
|
||||
|
||||
Full Gate All Clean Passes
|
||||
[Documentation] Verify the gate passes when all referenced bugs have clean tests
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} all_clean_passes
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} all-clean-passes-ok
|
||||
|
||||
TDD Tests In Both Behave And Robot
|
||||
[Documentation] Verify the gate checks tests in both .feature and .robot files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} both_behave_and_robot
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} both-behave-and-robot-ok
|
||||
|
||||
Diff Removal Is Required
|
||||
[Documentation] Verify the gate fails when PR diff has no expected-fail removal
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} diff_removal_required
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} diff-removal-required-ok
|
||||
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env python3
|
||||
"""TDD bug tag quality gate for bug fix PRs.
|
||||
|
||||
Enforces the TDD bug fix workflow rules described in CONTRIBUTING.md:
|
||||
|
||||
1. Parses the PR description for closing keywords that reference bug issues
|
||||
(``Closes #N``, ``Fixes #N``, ``Resolves #N``, ``ISSUES CLOSED: #N``).
|
||||
2. Searches the codebase for tests tagged ``@tdd_bug_N`` (Behave ``.feature``
|
||||
files) or ``tdd_bug_N`` (Robot ``.robot`` files).
|
||||
3. Verifies that every such test has had its ``@tdd_expected_fail`` /
|
||||
``tdd_expected_fail`` tag removed — the fix PR must remove the
|
||||
expected-fail marker as proof the bug is now fixed.
|
||||
|
||||
Exit codes:
|
||||
0 — All checks passed (or PR references no bugs).
|
||||
1 — One or more violations detected.
|
||||
|
||||
Usage:
|
||||
PR_DESCRIPTION="Fixes #42" python scripts/tdd_quality_gate.py
|
||||
|
||||
Or via nox::
|
||||
|
||||
PR_DESCRIPTION="Fixes #42" nox -s tdd_quality_gate
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Matches: Closes #N, Fixes #N, Resolves #N (case-insensitive)
|
||||
_CLOSING_KEYWORD_RE = re.compile(
|
||||
r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\b\s+#(\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches: ISSUES CLOSED: #N, #M, ...
|
||||
_ISSUES_CLOSED_RE = re.compile(
|
||||
r"ISSUES\s+CLOSED\s*:\s*((?:#\d+[\s,]*)+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Extracts individual issue numbers from the ISSUES CLOSED value
|
||||
_ISSUE_NUMBER_RE = re.compile(r"#(\d+)")
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _tag_token_pattern(tag: str) -> re.Pattern[str]:
|
||||
"""Return a compiled regex that matches ``tag`` as a full token."""
|
||||
escaped = re.escape(tag)
|
||||
return re.compile(rf"(?<![A-Za-z0-9_]){escaped}(?![A-Za-z0-9_])")
|
||||
|
||||
|
||||
def _contains_tag_token(content: str, tag: str) -> bool:
|
||||
"""Return True when ``tag`` appears as a full token in ``content``."""
|
||||
return _tag_token_pattern(tag).search(content) is not None
|
||||
|
||||
|
||||
def _collect_pr_diff(search_root: Path, base_ref: str) -> str:
|
||||
"""Collect unified diff between the PR branch and the base branch."""
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
if not isinstance(base_ref, str):
|
||||
raise TypeError(f"base_ref must be a str, got {type(base_ref).__name__}")
|
||||
if base_ref.strip() == "":
|
||||
raise ValueError("base_ref must not be empty")
|
||||
|
||||
ranges = (f"origin/{base_ref}...HEAD", f"{base_ref}...HEAD")
|
||||
for ref_range in ranges:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-color",
|
||||
ref_range,
|
||||
"--",
|
||||
"*.feature",
|
||||
"*.robot",
|
||||
],
|
||||
cwd=search_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return proc.stdout
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
continue
|
||||
|
||||
raise RuntimeError(
|
||||
"Unable to compute PR diff against base branch. "
|
||||
"Ensure git history for the base branch is available and retry."
|
||||
)
|
||||
|
||||
|
||||
def _diff_has_expected_fail_removal_for_bug(pr_diff: str, bug_number: int) -> bool:
|
||||
"""Return True when PR diff removes expected-fail for ``bug_number``."""
|
||||
if not isinstance(pr_diff, str):
|
||||
raise TypeError(f"pr_diff must be a str, got {type(pr_diff).__name__}")
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
|
||||
current_suffix = ""
|
||||
in_hunk = False
|
||||
file_has_bug_tag = False
|
||||
file_removed_expected_fail = False
|
||||
|
||||
for line in pr_diff.splitlines():
|
||||
if line.startswith("+++ "):
|
||||
if file_has_bug_tag and file_removed_expected_fail:
|
||||
return True
|
||||
raw_path = line[4:]
|
||||
if raw_path.startswith("b/"):
|
||||
raw_path = raw_path[2:]
|
||||
current_suffix = Path(raw_path).suffix.lower()
|
||||
in_hunk = False
|
||||
file_has_bug_tag = False
|
||||
file_removed_expected_fail = False
|
||||
continue
|
||||
|
||||
if current_suffix not in {".feature", ".robot"}:
|
||||
continue
|
||||
|
||||
if line.startswith("@@"):
|
||||
in_hunk = True
|
||||
continue
|
||||
|
||||
if not in_hunk:
|
||||
continue
|
||||
|
||||
if not line or line[0] not in {" ", "+", "-"}:
|
||||
continue
|
||||
|
||||
content = line[1:]
|
||||
if current_suffix == ".feature":
|
||||
bug_tag = f"@tdd_bug_{bug_number}"
|
||||
expected_fail_tag = "@tdd_expected_fail"
|
||||
else:
|
||||
bug_tag = f"tdd_bug_{bug_number}"
|
||||
expected_fail_tag = "tdd_expected_fail"
|
||||
|
||||
if _contains_tag_token(content, bug_tag):
|
||||
file_has_bug_tag = True
|
||||
if (
|
||||
line[0] == "-"
|
||||
and _contains_tag_token(content, expected_fail_tag)
|
||||
and _contains_tag_token(content, bug_tag)
|
||||
):
|
||||
file_removed_expected_fail = True
|
||||
|
||||
return file_has_bug_tag and file_removed_expected_fail
|
||||
|
||||
|
||||
def parse_bug_refs(pr_description: str) -> list[int]:
|
||||
"""Extract bug issue numbers from PR closing keywords.
|
||||
|
||||
Recognises ``Closes #N``, ``Fixes #N``, ``Resolves #N``
|
||||
(case-insensitive) and ``ISSUES CLOSED: #N, #M``.
|
||||
|
||||
Returns a deduplicated, sorted list of issue numbers.
|
||||
"""
|
||||
if not isinstance(pr_description, str):
|
||||
raise TypeError(
|
||||
f"pr_description must be a str, got {type(pr_description).__name__}"
|
||||
)
|
||||
|
||||
refs: set[int] = set()
|
||||
|
||||
# Standard closing keywords
|
||||
for match in _CLOSING_KEYWORD_RE.finditer(pr_description):
|
||||
num = int(match.group(1))
|
||||
if num > 0:
|
||||
refs.add(num)
|
||||
|
||||
# ISSUES CLOSED: #N, #M block
|
||||
for block_match in _ISSUES_CLOSED_RE.finditer(pr_description):
|
||||
block = block_match.group(1)
|
||||
for num_match in _ISSUE_NUMBER_RE.finditer(block):
|
||||
num = int(num_match.group(1))
|
||||
if num > 0:
|
||||
refs.add(num)
|
||||
|
||||
return sorted(refs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_tdd_tests(
|
||||
bug_number: int,
|
||||
search_root: Path,
|
||||
) -> list[Path]:
|
||||
"""Find test files tagged with ``@tdd_bug_<bug_number>``.
|
||||
|
||||
Searches ``.feature`` files for ``@tdd_bug_<N>`` and ``.robot``
|
||||
files for ``tdd_bug_<N>``.
|
||||
|
||||
Returns a list of paths that contain the tag.
|
||||
"""
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
|
||||
tag_behave = f"@tdd_bug_{bug_number}"
|
||||
tag_robot = f"tdd_bug_{bug_number}"
|
||||
matches: list[Path] = []
|
||||
|
||||
# Search .feature files
|
||||
for feature_file in sorted(search_root.rglob("*.feature")):
|
||||
try:
|
||||
content = feature_file.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if _contains_tag_token(content, tag_behave):
|
||||
matches.append(feature_file)
|
||||
|
||||
# Search .robot files
|
||||
for robot_file in sorted(search_root.rglob("*.robot")):
|
||||
try:
|
||||
content = robot_file.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if _contains_tag_token(content, tag_robot):
|
||||
matches.append(robot_file)
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag removal verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def check_expected_fail_removed(
|
||||
test_files: list[Path],
|
||||
bug_number: int,
|
||||
) -> list[str]:
|
||||
"""Verify ``@tdd_expected_fail`` has been removed from test files.
|
||||
|
||||
Returns a list of error messages for files that still contain the
|
||||
expected-fail tag.
|
||||
"""
|
||||
if not isinstance(test_files, list):
|
||||
raise TypeError(f"test_files must be a list, got {type(test_files).__name__}")
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
|
||||
errors: list[str] = []
|
||||
|
||||
for path in test_files:
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".feature":
|
||||
tag = "@tdd_expected_fail"
|
||||
elif suffix == ".robot":
|
||||
tag = "tdd_expected_fail"
|
||||
else:
|
||||
continue
|
||||
|
||||
if _contains_tag_token(content, tag):
|
||||
bug_tag_display = (
|
||||
f"@tdd_bug_{bug_number}"
|
||||
if suffix == ".feature"
|
||||
else f"tdd_bug_{bug_number}"
|
||||
)
|
||||
errors.append(
|
||||
f"Bug fix PR must remove the {tag} tag from tests tagged "
|
||||
f"{bug_tag_display}. "
|
||||
f"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main gate logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_quality_gate(
|
||||
pr_description: str,
|
||||
search_root: Path,
|
||||
*,
|
||||
pr_diff: str | None = None,
|
||||
base_ref: str = "master",
|
||||
) -> tuple[list[str], list[int]]:
|
||||
"""Run the full TDD quality gate.
|
||||
|
||||
Returns a ``(errors, bug_refs)`` tuple. An empty error list means all
|
||||
checks passed. ``bug_refs`` contains the parsed bug issue numbers so
|
||||
callers can avoid re-parsing the PR description.
|
||||
"""
|
||||
if not isinstance(pr_description, str):
|
||||
raise TypeError(
|
||||
f"pr_description must be a str, got {type(pr_description).__name__}"
|
||||
)
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
if pr_diff is not None and not isinstance(pr_diff, str):
|
||||
raise TypeError(f"pr_diff must be a str or None, got {type(pr_diff).__name__}")
|
||||
if not isinstance(base_ref, str):
|
||||
raise TypeError(f"base_ref must be a str, got {type(base_ref).__name__}")
|
||||
if base_ref.strip() == "":
|
||||
raise ValueError("base_ref must not be empty")
|
||||
|
||||
bug_refs = parse_bug_refs(pr_description)
|
||||
if not bug_refs:
|
||||
return [], bug_refs
|
||||
|
||||
if pr_diff is None:
|
||||
try:
|
||||
pr_diff = _collect_pr_diff(search_root, base_ref)
|
||||
except RuntimeError as exc:
|
||||
return [str(exc)], bug_refs
|
||||
|
||||
all_errors: list[str] = []
|
||||
|
||||
for bug_num in bug_refs:
|
||||
test_files = find_tdd_tests(bug_num, search_root)
|
||||
|
||||
if not test_files:
|
||||
all_errors.append(
|
||||
f"No TDD test found for bug #{bug_num}. "
|
||||
f"The TDD workflow requires a test tagged @tdd_bug_{bug_num} "
|
||||
f"to exist before the bug can be fixed. "
|
||||
f"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
continue
|
||||
|
||||
removal_errors = check_expected_fail_removed(test_files, bug_num)
|
||||
all_errors.extend(removal_errors)
|
||||
|
||||
# Only check the diff when the file-level check found no tag issues;
|
||||
# otherwise the diff error would be redundant.
|
||||
if not removal_errors and not _diff_has_expected_fail_removal_for_bug(
|
||||
pr_diff, bug_num
|
||||
):
|
||||
all_errors.append(
|
||||
"No removal of @tdd_expected_fail / tdd_expected_fail detected "
|
||||
f"in PR diff for bug #{bug_num}. "
|
||||
"The bug fix PR must remove the expected-fail tag in this branch. "
|
||||
"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
|
||||
return all_errors, bug_refs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point. Reads PR_DESCRIPTION from the environment."""
|
||||
pr_description = os.environ.get("PR_DESCRIPTION", "")
|
||||
base_ref = os.environ.get("PR_BASE_REF", "master")
|
||||
search_root = Path.cwd()
|
||||
|
||||
errors, bug_refs = run_quality_gate(pr_description, search_root, base_ref=base_ref)
|
||||
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if bug_refs:
|
||||
print(f"TDD quality gate passed for bug(s): {bug_refs}")
|
||||
else:
|
||||
print("TDD quality gate: no bug references found in PR description (pass).")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -54,9 +54,17 @@ class Agent(ABC):
|
||||
def process_message_sync(
|
||||
self, message: Any, context: dict[str, Any] | None = None
|
||||
) -> Any:
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.process_message(message, context or {})
|
||||
)
|
||||
try:
|
||||
_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
_loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return _loop.run_until_complete(
|
||||
self.process_message(message, context or {})
|
||||
)
|
||||
finally:
|
||||
if not _loop.is_running():
|
||||
_loop.close()
|
||||
|
||||
@abstractmethod
|
||||
async def process_message(
|
||||
|
||||
@@ -527,8 +527,8 @@ class PlanGenerationGraph:
|
||||
)
|
||||
validation = str(result)
|
||||
|
||||
# Simple validation check (in real implementation, parse the LLM response)
|
||||
is_valid = "PASS" in validation.upper() or len(all_code) > 10
|
||||
# Reliance on the LLM response to determine pass/fail.
|
||||
is_valid = "PASS" in validation.upper()
|
||||
|
||||
return {
|
||||
"validation_result": {
|
||||
|
||||
@@ -1183,6 +1183,14 @@ class ConfigService:
|
||||
project_root: Path | _AutoDiscover | None = _AUTO_DISCOVER,
|
||||
) -> None:
|
||||
self._config_dir: Path = config_dir or _DEFAULT_CONFIG_DIR
|
||||
# Honour CLEVERAGENTS_CONFIG_PATH env var when no explicit path is given.
|
||||
# This allows the --config-path CLI flag (which sets the env var in
|
||||
# main_callback) to propagate into all ConfigService instances created
|
||||
# during subcommand execution (ADR-024 §Resolution Chain).
|
||||
if config_path is None:
|
||||
_env_config_path = os.environ.get("CLEVERAGENTS_CONFIG_PATH", "").strip()
|
||||
if _env_config_path:
|
||||
config_path = Path(_env_config_path)
|
||||
self._config_path: Path = config_path or (self._config_dir / "config.toml")
|
||||
self._event_bus = event_bus
|
||||
# ``_AUTO_DISCOVER`` means "auto-discover"; ``None`` means "no project root".
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Based on ADR-009: CLI Framework using Typer.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Annotated, Any
|
||||
@@ -249,6 +250,13 @@ def _print_basic_help() -> None:
|
||||
"""Print a lightweight help message without heavy imports."""
|
||||
typer.echo("CleverAgents - AI-powered development assistant (actor-first)")
|
||||
typer.echo("Usage: cleveragents [OPTIONS] COMMAND [ARGS]...")
|
||||
typer.echo("\nGlobal options:")
|
||||
typer.echo(" --data-dir PATH Override global data directory")
|
||||
typer.echo(" --config-path PATH Override global configuration file path")
|
||||
typer.echo(" -v Increase log verbosity (repeatable)")
|
||||
typer.echo(" --format -f FMT Output format: rich/color/table/plain/json/yaml")
|
||||
typer.echo(" --version Show version and exit")
|
||||
typer.echo(" --help -h Show this message and exit")
|
||||
typer.echo("\nCommon commands:")
|
||||
typer.echo(" project Project management")
|
||||
typer.echo(" actor context Actor context management")
|
||||
@@ -272,6 +280,21 @@ def _print_basic_help() -> None:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verbosity (-v) → log level mapping (ADR-021 §Global CLI Flags)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mapping: 0 = silent (CRITICAL), 1 = ERROR, 2 = WARNING, 3 = INFO,
|
||||
# 4 = DEBUG, 5+ = DEBUG (TRACE not available in Python stdlib).
|
||||
_VERBOSITY_LOG_LEVELS: tuple[str, ...] = (
|
||||
"CRITICAL", # 0 — no -v flag (silent)
|
||||
"ERROR", # 1 — -v
|
||||
"WARNING", # 2 — -vv
|
||||
"INFO", # 3 — -vvv
|
||||
"DEBUG", # 4 — -vvvv
|
||||
"DEBUG", # 5+ — -vvvvv (TRACE mapped to DEBUG)
|
||||
)
|
||||
|
||||
|
||||
def version_callback(value: bool) -> None:
|
||||
"""Handle --version flag."""
|
||||
if value:
|
||||
@@ -316,20 +339,98 @@ def main_callback(
|
||||
),
|
||||
),
|
||||
] = OutputFormat.RICH,
|
||||
data_dir: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
"--data-dir",
|
||||
help=(
|
||||
"Override the global data directory for this invocation "
|
||||
"(database, caches, sessions, logs). "
|
||||
"Overrides CLEVERAGENTS_DATA_DIR and core.data-dir config key."
|
||||
),
|
||||
metavar="PATH",
|
||||
),
|
||||
] = None,
|
||||
config_path: Annotated[
|
||||
Path | None,
|
||||
typer.Option(
|
||||
"--config-path",
|
||||
help=(
|
||||
"Override the global configuration file path for this invocation. "
|
||||
"Overrides CLEVERAGENTS_CONFIG_PATH and the default config location."
|
||||
),
|
||||
metavar="PATH",
|
||||
),
|
||||
] = None,
|
||||
verbose: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"-v",
|
||||
count=True,
|
||||
help=(
|
||||
"Increase log verbosity (repeatable). "
|
||||
"No flag = silent; -v = ERROR; -vv = WARN; "
|
||||
"-vvv = INFO; -vvvv = DEBUG; -vvvvv = TRACE."
|
||||
),
|
||||
),
|
||||
] = 0,
|
||||
) -> None:
|
||||
"""CleverAgents - AI-powered development assistant."""
|
||||
# Suppress debug-level logs on stdout for ALL commands so machine-readable
|
||||
# output formats (json, yaml, plain) receive clean stdout. Commands that
|
||||
# need verbose logging can override this after parsing --log-level flags.
|
||||
from cleveragents.config.logging import configure_structlog
|
||||
|
||||
configure_structlog(log_level="WARNING")
|
||||
# -----------------------------------------------------------------------
|
||||
# Validate and wire --data-dir
|
||||
# -----------------------------------------------------------------------
|
||||
if data_dir is not None:
|
||||
resolved_data_dir = data_dir.resolve()
|
||||
if resolved_data_dir.exists() and not resolved_data_dir.is_dir():
|
||||
get_err_console().print(
|
||||
f"[red]Error: --data-dir '{data_dir}' exists but is not"
|
||||
" a directory.[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
# Wire: set env var so Settings and other components pick it up.
|
||||
# Settings.__new__ reads CLEVERAGENTS_DATA_DIR on first access, so
|
||||
# setting the env var before any Settings-reading code runs is the
|
||||
# correct (and non-destructive) way to propagate the CLI override.
|
||||
os.environ["CLEVERAGENTS_DATA_DIR"] = str(resolved_data_dir)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Validate and wire --config-path
|
||||
# -----------------------------------------------------------------------
|
||||
if config_path is not None:
|
||||
resolved_config_path = config_path.resolve()
|
||||
if not resolved_config_path.exists():
|
||||
get_err_console().print(
|
||||
f"[red]Error: --config-path '{config_path}' does not exist.[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
if not resolved_config_path.is_file():
|
||||
get_err_console().print(
|
||||
f"[red]Error: --config-path '{config_path}' is not a file.[/red]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
# Wire: set env var so ConfigService instances created later use this path
|
||||
os.environ["CLEVERAGENTS_CONFIG_PATH"] = str(resolved_config_path)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Configure log verbosity from -v count (ADR-021 §Global CLI Flags)
|
||||
# -----------------------------------------------------------------------
|
||||
log_level = _VERBOSITY_LOG_LEVELS[min(verbose, len(_VERBOSITY_LOG_LEVELS) - 1)]
|
||||
configure_structlog(log_level=log_level)
|
||||
|
||||
_register_subcommands()
|
||||
|
||||
# Store the selected output format in the Typer context so all subcommands
|
||||
# can read it via ctx.obj["format"] without needing their own --format flag.
|
||||
# -----------------------------------------------------------------------
|
||||
# Store all global options in ctx.obj for subcommand access
|
||||
# -----------------------------------------------------------------------
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["format"] = fmt.value
|
||||
ctx.obj["data_dir"] = str(data_dir.resolve()) if data_dir is not None else None
|
||||
ctx.obj["config_path"] = (
|
||||
str(config_path.resolve()) if config_path is not None else None
|
||||
)
|
||||
ctx.obj["verbose"] = verbose
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -760,8 +861,13 @@ def main(args: list[str] | None = None) -> int:
|
||||
return 130
|
||||
except Exception as e:
|
||||
from cleveragents.core.error_handling import classify_error, wrap_unexpected
|
||||
from cleveragents.shared.redaction import redact_value
|
||||
|
||||
err_console = get_err_console()
|
||||
# Always print the original exception type/message to stderr so that
|
||||
# actionable details (e.g. "No such option: --flag") remain visible
|
||||
# even when the log level is set to CRITICAL (structlog suppressed).
|
||||
err_console.print(f"[dim]{type(e).__name__}: {redact_value(str(e))}[/dim]")
|
||||
safe = wrap_unexpected(e)
|
||||
info = classify_error(safe)
|
||||
err_console.print(
|
||||
|
||||
@@ -224,7 +224,10 @@ class RxPyLangGraphBridge:
|
||||
)
|
||||
|
||||
def create_future_task(msg: StreamMessage) -> asyncio.Future[StreamMessage]:
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
task: asyncio.Future[StreamMessage] = asyncio.ensure_future(
|
||||
execute_graph(msg), loop=loop
|
||||
)
|
||||
|
||||
@@ -106,7 +106,10 @@ class Node: # pylint: disable=too-many-instance-attributes
|
||||
|
||||
async def execute(self, state: GraphState) -> dict[str, Any]: # pylint: disable=too-many-branches
|
||||
self.execution_count += 1
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
start_time = loop.time()
|
||||
try:
|
||||
state_updates = {"current_node": self.name}
|
||||
@@ -271,7 +274,10 @@ class Node: # pylint: disable=too-many-instance-attributes
|
||||
if asyncio.iscoroutinefunction(fn):
|
||||
result = await fn(state)
|
||||
else:
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
result = await loop.run_in_executor(None, lambda: fn(state))
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from rx.scheduler.eventloop import AsyncIOScheduler # type: ignore[attr-defined]
|
||||
@@ -86,7 +86,7 @@ class RouteBridge:
|
||||
last_updated = state.metadata.get("last_updated")
|
||||
if (
|
||||
last_updated
|
||||
and (asyncio.get_event_loop().time() - last_updated) > idle_threshold
|
||||
and (time.time() - last_updated) > idle_threshold
|
||||
):
|
||||
return True
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user