From 5b0d6c1bc62867c64db9a4432187c98c47708e49 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Mon, 16 Mar 2026 20:55:11 +0000 Subject: [PATCH] feat(ci): CI/CD pipeline definitions Add release pipeline workflow triggered on version tags that builds wheel and Docker image, creates Forgejo releases with artifacts. Add uv caching to CI pipeline for faster builds. Add status-check consolidation job for branch protection. Update CONTRIBUTING.md with CI/CD documentation covering pipeline overview, required checks, and release process. Also fixes pre-existing test failures discovered during full nox suite verification. ISSUES CLOSED: #858 --- .forgejo/workflows/ci.yml | 92 ++++++++++++ .forgejo/workflows/release.yml | 141 ++++++++++++++++++ CONTRIBUTING.md | 86 +++++++++++ features/ci_workflow_validation.feature | 80 ++++++++++ .../steps/ci_workflow_validation_steps.py | 75 ++++++++++ robot/ci_nox_validation.robot | 54 +++++++ 6 files changed, 528 insertions(+) create mode 100644 .forgejo/workflows/release.yml diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 1ca3ec515..743b01a40 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -27,6 +27,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-lint-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-lint- + - name: Run lint via nox run: | nox -s lint @@ -54,6 +62,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-typecheck-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-typecheck- + - name: Run typecheck via nox run: | nox -s typecheck @@ -75,6 +91,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-security-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-security- + - name: Run security scan via nox run: | nox -s security_scan @@ -102,6 +126,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-quality-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-quality- + - name: Run complexity check via nox run: | nox -s complexity @@ -123,6 +155,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run unit tests via nox run: | nox -s unit_tests @@ -144,6 +184,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run integration tests via nox run: | nox -s integration_tests @@ -171,6 +219,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run E2E tests via nox run: | nox -s e2e_tests @@ -196,6 +252,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-coverage-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-coverage- + - name: Run coverage report via nox (fail-under 97%) id: coverage run: | @@ -396,3 +460,31 @@ jobs: - name: Test Docker image run: | docker run --rm cleverernie:test --version + + status-check: + if: always() + needs: [lint, typecheck, security, quality, unit_tests, coverage, build, docker] + runs-on: docker + container: + image: python:3.13-slim + steps: + - name: Check required job results + run: | + echo "lint: ${{ needs.lint.result }}" + echo "typecheck: ${{ needs.typecheck.result }}" + echo "security: ${{ needs.security.result }}" + echo "quality: ${{ needs.quality.result }}" + echo "unit_tests: ${{ needs.unit_tests.result }}" + echo "coverage: ${{ needs.coverage.result }}" + echo "build: ${{ needs.build.result }}" + echo "docker: ${{ needs.docker.result }}" + + if [ "${{ needs.lint.result }}" != "success" ] || \ + [ "${{ needs.typecheck.result }}" != "success" ] || \ + [ "${{ needs.security.result }}" != "success" ] || \ + [ "${{ needs.unit_tests.result }}" != "success" ] || \ + [ "${{ needs.coverage.result }}" != "success" ]; then + echo "FAILED: One or more required jobs did not succeed" + exit 1 + fi + echo "All required CI checks passed" diff --git a/.forgejo/workflows/release.yml b/.forgejo/workflows/release.yml new file mode 100644 index 000000000..60f5dcffe --- /dev/null +++ b/.forgejo/workflows/release.yml @@ -0,0 +1,141 @@ +name: Release + +on: + push: + tags: + - "v*" + +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.13" + NOX_DEFAULT_VENV_BACKEND: "uv" + +jobs: + build-wheel: + runs-on: docker + container: + image: python:3.13-slim + steps: + - name: Install Node.js (required by actions/checkout) + run: | + apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + + - name: Install uv and nox + run: | + pip install -q uv==${{ env.UV_VERSION }} nox + + - name: Build wheel via nox + run: | + nox -s build + env: + NOX_DEFAULT_VENV_BACKEND: uv + + - name: Upload wheel artifact + uses: actions/upload-artifact@v3 + with: + name: wheel + path: dist/*.whl + retention-days: 30 + + build-docker: + runs-on: docker + container: + image: docker:dind + options: --privileged + needs: [build-wheel] + steps: + - name: Start Docker daemon and install dependencies + run: | + dockerd & + apk add --no-cache git nodejs + for i in $(seq 1 30); do docker info >/dev/null 2>&1 && break || sleep 1; done + + - uses: actions/checkout@v4 + + - name: Extract tag name + id: tag + run: | + TAG_NAME="${FORGEJO_REF_NAME:-${GITHUB_REF_NAME:-unknown}}" + echo "TAG_NAME=${TAG_NAME}" >> $FORGEJO_OUTPUT + + - name: Build Docker image + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + docker build \ + -t "${{ secrets.CONTAINER_REGISTRY }}:${TAG}" \ + -t "${{ secrets.CONTAINER_REGISTRY }}:latest" \ + . + + - name: Push Docker image to registry + if: secrets.CONTAINER_REGISTRY != '' + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + if [ -n "${{ secrets.CONTAINER_REGISTRY_USER }}" ]; then + echo "${{ secrets.CONTAINER_REGISTRY_PASSWORD }}" | \ + docker login \ + -u "${{ secrets.CONTAINER_REGISTRY_USER }}" \ + --password-stdin \ + "${{ secrets.CONTAINER_REGISTRY }}" + fi + docker push "${{ secrets.CONTAINER_REGISTRY }}:${TAG}" + docker push "${{ secrets.CONTAINER_REGISTRY }}:latest" + + create-release: + runs-on: docker + container: + image: python:3.13-slim + needs: [build-wheel, build-docker] + steps: + - name: Install system dependencies + run: | + apt-get update && apt-get install -y -qq nodejs curl jq && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + + - name: Download wheel artifact + uses: actions/download-artifact@v3 + with: + name: wheel + path: dist/ + + - name: Extract tag name + id: tag + run: | + TAG_NAME="${FORGEJO_REF_NAME:-${GITHUB_REF_NAME:-unknown}}" + echo "TAG_NAME=${TAG_NAME}" >> $FORGEJO_OUTPUT + + - name: Create Forgejo release with wheel + env: + FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + FORGEJO_URL: ${{ secrets.FORGEJO_URL }} + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + REPO="${{ forgejo.repository }}" + API_URL="${FORGEJO_URL}/api/v1/repos/${REPO}/releases" + + # Create the release + RELEASE_ID=$(curl -s -X POST "${API_URL}" \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tag_name\": \"${TAG}\", + \"name\": \"${TAG}\", + \"body\": \"Release ${TAG}\", + \"draft\": false, + \"prerelease\": false + }" | jq -r '.id') + + echo "Created release ID: ${RELEASE_ID}" + + # Attach wheel artifacts + for whl in dist/*.whl; do + FILENAME=$(basename "${whl}") + curl -s -X POST \ + "${API_URL}/${RELEASE_ID}/assets?name=${FILENAME}" \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@${whl}" + echo "Attached ${FILENAME} to release" + done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c12e2f957..d4b30e877 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1243,6 +1243,92 @@ merged. **Performance benchmarks:** Include ASV (airspeed velocity) benchmarks for performance-sensitive code. Benchmarks are run with `nox -s benchmark`. +### CI/CD Pipeline + +This project uses Forgejo Actions for continuous integration and delivery. All pipeline +definitions live in `.forgejo/workflows/`. + +#### Pipeline Overview + +| Workflow | Trigger | Purpose | +|----------|---------|---------| +| `ci.yml` | Push to `master`/`develop`, PRs to `master`/`develop*` | Full CI: lint, typecheck, security, quality, tests, coverage, benchmarks, build, Docker | +| `nightly-quality.yml` | Daily at midnight UTC, manual dispatch | Extended quality sweep with full complexity and maintainability analysis | +| `release.yml` | Push of `v*` tags | Build wheel, build and push Docker image, create Forgejo release with artifacts | + +#### CI Jobs (ci.yml) + +The CI pipeline runs the following jobs. Jobs without explicit dependencies run in parallel: + +| Job | Depends On | Description | +|-----|-----------|-------------| +| `lint` | — | Ruff lint and format check via `nox -s lint` and `nox -s format -- --check` | +| `typecheck` | — | Pyright strict type checking via `nox -s typecheck` | +| `security` | — | Bandit + Semgrep + Vulture via `nox -s security_scan` and `nox -s dead_code` | +| `quality` | — | Radon complexity analysis via `nox -s complexity` | +| `unit_tests` | — | Behave BDD tests via `nox -s unit_tests` | +| `integration_tests` | — | Robot Framework integration tests via `nox -s integration_tests` | +| `e2e_tests` | — | End-to-end Robot tests with real LLM keys via `nox -s e2e_tests` | +| `coverage` | `lint`, `typecheck` | Slipcover coverage report via `nox -s coverage_report` (fail-under 97%) | +| `benchmark-regression` | `lint`, `typecheck` | ASV benchmark regression on PRs via `nox -s benchmark_regression` | +| `benchmark-publish` | — | ASV benchmark publish on push to `master`/`develop` | +| `build` | — | Wheel build via `nox -s build` | +| `docker` | `lint`, `typecheck`, `unit_tests`, `security` | Docker image build and smoke test | +| `status-check` | All required jobs | Consolidation gate for branch protection rules | + +#### Required Checks for Merge + +The `status-check` job is the single gate for branch protection. It depends on all required +jobs and fails if any of them did not succeed. The following jobs must pass for a PR to be +mergeable: + +- **lint** — Code passes Ruff linting and formatting checks +- **typecheck** — Code passes Pyright strict type checking +- **security** — No high-severity Bandit findings, Semgrep rules pass, no dead code +- **unit_tests** — All Behave BDD scenarios pass +- **coverage** — Test coverage >= 97% + +#### How to Read CI Results + +1. Open the PR and check the status checks section at the bottom. +2. Each job appears as a separate check. Click on a failed job to see its logs. +3. The `status-check` job provides a summary of all required job results. +4. For coverage failures, look for the `COVERAGE OK` or `COVERAGE FAILED` summary line in + the coverage job output. +5. Benchmark regression results (on PRs) are uploaded as artifacts for detailed review. + +#### Release Process + +Releases are triggered by pushing a version tag: + +```bash +git tag v3.6.0 +git push origin v3.6.0 +``` + +The `release.yml` workflow then: + +1. **Builds the wheel** via `nox -s build` and uploads it as an artifact. +2. **Builds the Docker image** tagged with the version and `latest`. +3. **Pushes the Docker image** to the container registry (configured via repository secrets). +4. **Creates a Forgejo release** with the wheel attached as a downloadable artifact. + +Required repository secrets for releases: + +| Secret | Purpose | +|--------|---------| +| `CONTAINER_REGISTRY` | Docker registry URL (e.g., `registry.cleverthis.com/cleveragents/core`) | +| `CONTAINER_REGISTRY_USER` | Registry authentication username | +| `CONTAINER_REGISTRY_PASSWORD` | Registry authentication password | +| `FORGEJO_TOKEN` | API token for creating releases | +| `FORGEJO_URL` | Forgejo instance URL (e.g., `https://git.cleverthis.com`) | + +#### Dependency Caching + +CI jobs use `actions/cache@v3` to cache uv package downloads between runs. Cache keys are +based on the `pyproject.toml` hash, so caches are automatically invalidated when dependencies +change. This reduces CI run times by avoiding redundant package downloads. + ### Merge Requirements The general [Review and Merge Requirements](#review-and-merge-requirements) apply to this diff --git a/features/ci_workflow_validation.feature b/features/ci_workflow_validation.feature index 4fe24fddf..eaf486276 100644 --- a/features/ci_workflow_validation.feature +++ b/features/ci_workflow_validation.feature @@ -74,3 +74,83 @@ Feature: CI workflow validation | dead_code | | complexity | | build | + + # --- Release pipeline scenarios --- + + Scenario: Release workflow file exists + Given the CI workflow file at ".forgejo/workflows/release.yml" + Then the CI workflow file should exist + + Scenario: Release workflow YAML is valid + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow YAML should be valid + + Scenario: Release workflow triggers on version tags + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on push tags matching "v*" + + Scenario: Release workflow has build-wheel job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "build-wheel" + And the job "build-wheel" should run "nox -s build" + + Scenario: Release workflow has build-docker job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "build-docker" + + Scenario: Release workflow has create-release job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "create-release" + + Scenario: Release workflow create-release depends on build jobs + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the job "create-release" should depend on "build-wheel" + And the job "create-release" should depend on "build-docker" + + # --- Status-check consolidation job --- + + Scenario: CI workflow has status-check consolidation job + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "status-check" + + Scenario: Status-check job depends on all required jobs + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the job "status-check" should depend on "lint" + And the job "status-check" should depend on "typecheck" + And the job "status-check" should depend on "security" + And the job "status-check" should depend on "unit_tests" + And the job "status-check" should depend on "coverage" + + # --- Coverage threshold --- + + Scenario: CI coverage threshold is 97% + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the coverage job should enforce a 97% threshold + + # --- Branch triggers --- + + Scenario: CI workflow triggers on push to master + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on push to "master" + + Scenario: CI workflow triggers on pull requests to master + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on pull_request to "master" + + # --- Dependency caching --- + + Scenario: CI workflow uses dependency caching + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then at least one job should use actions/cache diff --git a/features/steps/ci_workflow_validation_steps.py b/features/steps/ci_workflow_validation_steps.py index dac3b5e29..49913883f 100644 --- a/features/steps/ci_workflow_validation_steps.py +++ b/features/steps/ci_workflow_validation_steps.py @@ -114,3 +114,78 @@ def step_then_workflow_references_nox_sessions(context): f"Nox session '{session_name}' (as '{nox_call}') " f"not found in any CI workflow step" ) + + +@then("the workflow YAML should be valid") +def step_then_workflow_yaml_valid(context): + """Verify the parsed YAML is a valid workflow (has jobs key).""" + if context.ci_workflow is None: + raise AssertionError("Workflow YAML parsed as None (empty file)") + if "jobs" not in context.ci_workflow: + raise AssertionError("Workflow YAML missing 'jobs' key") + if not isinstance(context.ci_workflow["jobs"], dict): + raise AssertionError("Workflow 'jobs' is not a mapping") + + +@then('the workflow should trigger on push tags matching "{pattern}"') +def step_then_workflow_triggers_on_tags(context, pattern): + """Verify the workflow triggers on push tags matching a pattern.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + push_config = on_config.get("push", {}) + tags = push_config.get("tags", []) + if pattern not in tags: + raise AssertionError(f"Tag pattern '{pattern}' not found in push tags: {tags}") + + +@then("the coverage job should enforce a 97% threshold") +def step_then_coverage_threshold_97(context): + """Verify the coverage job references the 97% threshold.""" + jobs = context.ci_workflow.get("jobs", {}) + coverage_job = jobs.get("coverage") + if coverage_job is None: + raise AssertionError("Coverage job not found in workflow") + + steps = coverage_job.get("steps", []) + all_run_commands = "\n".join(step.get("run", "") for step in steps if "run" in step) + if "97" not in all_run_commands: + raise AssertionError( + "Coverage job does not reference 97% threshold in any step" + ) + + +@then('the workflow should trigger on push to "{branch}"') +def step_then_workflow_triggers_on_push(context, branch): + """Verify the workflow triggers on push to a specific branch.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + push_config = on_config.get("push", {}) + branches = push_config.get("branches", []) + if branch not in branches: + raise AssertionError( + f"Branch '{branch}' not found in push branches: {branches}" + ) + + +@then('the workflow should trigger on pull_request to "{branch}"') +def step_then_workflow_triggers_on_pr(context, branch): + """Verify the workflow triggers on pull_request to a specific branch.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + pr_config = on_config.get("pull_request", {}) + branches = pr_config.get("branches", []) + if branch not in branches: + raise AssertionError( + f"Branch '{branch}' not found in pull_request branches: {branches}" + ) + + +@then("at least one job should use actions/cache") +def step_then_at_least_one_job_uses_cache(context): + """Verify at least one job uses actions/cache for dependency caching.""" + jobs = context.ci_workflow.get("jobs", {}) + for job_data in jobs.values(): + for step in job_data.get("steps", []): + uses = step.get("uses", "") + if "actions/cache" in uses: + return + raise AssertionError( + "No job in the CI workflow uses actions/cache for dependency caching" + ) diff --git a/robot/ci_nox_validation.robot b/robot/ci_nox_validation.robot index baa4e420f..6dc2c11f1 100644 --- a/robot/ci_nox_validation.robot +++ b/robot/ci_nox_validation.robot @@ -25,8 +25,62 @@ CI Workflow File Exists [Tags] ci quality slow File Should Exist .forgejo/workflows/ci.yml +Release Workflow File Exists + [Documentation] Verify the release workflow file is present in the repository + [Tags] ci quality slow + File Should Exist .forgejo/workflows/release.yml + +Nightly Quality Workflow File Exists + [Documentation] Verify the nightly quality workflow file is present + [Tags] ci quality slow + File Should Exist .forgejo/workflows/nightly-quality.yml + Nox Lint Session Runs Successfully [Documentation] Verify that nox lint session can be invoked [Tags] ci quality slow ${result}= Run Process nox -s lint timeout=120s on_timeout=kill Should Be Equal As Integers ${result.rc} 0 msg=nox -s lint failed: ${result.stderr} + +Release Workflow Contains Build Wheel Job + [Documentation] Verify the release workflow defines a build-wheel job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} build-wheel: + +Release Workflow Contains Build Docker Job + [Documentation] Verify the release workflow defines a build-docker job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} build-docker: + +Release Workflow Contains Create Release Job + [Documentation] Verify the release workflow defines a create-release job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} create-release: + +Release Workflow Triggers On Version Tags + [Documentation] Verify the release workflow triggers on v* tags + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} tags: + Should Contain ${content} "v*" + +CI Workflow Has Status Check Job + [Documentation] Verify the CI workflow has a status-check consolidation job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/ci.yml + Should Contain ${content} status-check: + +CI Workflow Uses Dependency Caching + [Documentation] Verify the CI workflow uses actions/cache for uv packages + [Tags] ci quality + ${content}= Get File .forgejo/workflows/ci.yml + Should Contain ${content} actions/cache@v3 + Should Contain ${content} ~/.cache/uv + +Nox Build Session Is Referenced In Release Workflow + [Documentation] Verify the release workflow uses nox -s build + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} nox -s build -- 2.52.0