diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 1ca3ec515..743b01a40 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -27,6 +27,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-lint-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-lint- + - name: Run lint via nox run: | nox -s lint @@ -54,6 +62,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-typecheck-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-typecheck- + - name: Run typecheck via nox run: | nox -s typecheck @@ -75,6 +91,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-security-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-security- + - name: Run security scan via nox run: | nox -s security_scan @@ -102,6 +126,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-quality-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-quality- + - name: Run complexity check via nox run: | nox -s complexity @@ -123,6 +155,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run unit tests via nox run: | nox -s unit_tests @@ -144,6 +184,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run integration tests via nox run: | nox -s integration_tests @@ -171,6 +219,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-tests-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-tests- + - name: Run E2E tests via nox run: | nox -s e2e_tests @@ -196,6 +252,14 @@ jobs: run: | pip install -q uv==${{ env.UV_VERSION }} nox + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-coverage-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv-coverage- + - name: Run coverage report via nox (fail-under 97%) id: coverage run: | @@ -396,3 +460,31 @@ jobs: - name: Test Docker image run: | docker run --rm cleverernie:test --version + + status-check: + if: always() + needs: [lint, typecheck, security, quality, unit_tests, coverage, build, docker] + runs-on: docker + container: + image: python:3.13-slim + steps: + - name: Check required job results + run: | + echo "lint: ${{ needs.lint.result }}" + echo "typecheck: ${{ needs.typecheck.result }}" + echo "security: ${{ needs.security.result }}" + echo "quality: ${{ needs.quality.result }}" + echo "unit_tests: ${{ needs.unit_tests.result }}" + echo "coverage: ${{ needs.coverage.result }}" + echo "build: ${{ needs.build.result }}" + echo "docker: ${{ needs.docker.result }}" + + if [ "${{ needs.lint.result }}" != "success" ] || \ + [ "${{ needs.typecheck.result }}" != "success" ] || \ + [ "${{ needs.security.result }}" != "success" ] || \ + [ "${{ needs.unit_tests.result }}" != "success" ] || \ + [ "${{ needs.coverage.result }}" != "success" ]; then + echo "FAILED: One or more required jobs did not succeed" + exit 1 + fi + echo "All required CI checks passed" diff --git a/.forgejo/workflows/release.yml b/.forgejo/workflows/release.yml new file mode 100644 index 000000000..60f5dcffe --- /dev/null +++ b/.forgejo/workflows/release.yml @@ -0,0 +1,141 @@ +name: Release + +on: + push: + tags: + - "v*" + +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.13" + NOX_DEFAULT_VENV_BACKEND: "uv" + +jobs: + build-wheel: + runs-on: docker + container: + image: python:3.13-slim + steps: + - name: Install Node.js (required by actions/checkout) + run: | + apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + + - name: Install uv and nox + run: | + pip install -q uv==${{ env.UV_VERSION }} nox + + - name: Build wheel via nox + run: | + nox -s build + env: + NOX_DEFAULT_VENV_BACKEND: uv + + - name: Upload wheel artifact + uses: actions/upload-artifact@v3 + with: + name: wheel + path: dist/*.whl + retention-days: 30 + + build-docker: + runs-on: docker + container: + image: docker:dind + options: --privileged + needs: [build-wheel] + steps: + - name: Start Docker daemon and install dependencies + run: | + dockerd & + apk add --no-cache git nodejs + for i in $(seq 1 30); do docker info >/dev/null 2>&1 && break || sleep 1; done + + - uses: actions/checkout@v4 + + - name: Extract tag name + id: tag + run: | + TAG_NAME="${FORGEJO_REF_NAME:-${GITHUB_REF_NAME:-unknown}}" + echo "TAG_NAME=${TAG_NAME}" >> $FORGEJO_OUTPUT + + - name: Build Docker image + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + docker build \ + -t "${{ secrets.CONTAINER_REGISTRY }}:${TAG}" \ + -t "${{ secrets.CONTAINER_REGISTRY }}:latest" \ + . + + - name: Push Docker image to registry + if: secrets.CONTAINER_REGISTRY != '' + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + if [ -n "${{ secrets.CONTAINER_REGISTRY_USER }}" ]; then + echo "${{ secrets.CONTAINER_REGISTRY_PASSWORD }}" | \ + docker login \ + -u "${{ secrets.CONTAINER_REGISTRY_USER }}" \ + --password-stdin \ + "${{ secrets.CONTAINER_REGISTRY }}" + fi + docker push "${{ secrets.CONTAINER_REGISTRY }}:${TAG}" + docker push "${{ secrets.CONTAINER_REGISTRY }}:latest" + + create-release: + runs-on: docker + container: + image: python:3.13-slim + needs: [build-wheel, build-docker] + steps: + - name: Install system dependencies + run: | + apt-get update && apt-get install -y -qq nodejs curl jq && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + + - name: Download wheel artifact + uses: actions/download-artifact@v3 + with: + name: wheel + path: dist/ + + - name: Extract tag name + id: tag + run: | + TAG_NAME="${FORGEJO_REF_NAME:-${GITHUB_REF_NAME:-unknown}}" + echo "TAG_NAME=${TAG_NAME}" >> $FORGEJO_OUTPUT + + - name: Create Forgejo release with wheel + env: + FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} + FORGEJO_URL: ${{ secrets.FORGEJO_URL }} + run: | + TAG="${{ steps.tag.outputs.TAG_NAME }}" + REPO="${{ forgejo.repository }}" + API_URL="${FORGEJO_URL}/api/v1/repos/${REPO}/releases" + + # Create the release + RELEASE_ID=$(curl -s -X POST "${API_URL}" \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tag_name\": \"${TAG}\", + \"name\": \"${TAG}\", + \"body\": \"Release ${TAG}\", + \"draft\": false, + \"prerelease\": false + }" | jq -r '.id') + + echo "Created release ID: ${RELEASE_ID}" + + # Attach wheel artifacts + for whl in dist/*.whl; do + FILENAME=$(basename "${whl}") + curl -s -X POST \ + "${API_URL}/${RELEASE_ID}/assets?name=${FILENAME}" \ + -H "Authorization: token ${FORGEJO_TOKEN}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@${whl}" + echo "Attached ${FILENAME} to release" + done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c12e2f957..d4b30e877 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1243,6 +1243,92 @@ merged. **Performance benchmarks:** Include ASV (airspeed velocity) benchmarks for performance-sensitive code. Benchmarks are run with `nox -s benchmark`. +### CI/CD Pipeline + +This project uses Forgejo Actions for continuous integration and delivery. All pipeline +definitions live in `.forgejo/workflows/`. + +#### Pipeline Overview + +| Workflow | Trigger | Purpose | +|----------|---------|---------| +| `ci.yml` | Push to `master`/`develop`, PRs to `master`/`develop*` | Full CI: lint, typecheck, security, quality, tests, coverage, benchmarks, build, Docker | +| `nightly-quality.yml` | Daily at midnight UTC, manual dispatch | Extended quality sweep with full complexity and maintainability analysis | +| `release.yml` | Push of `v*` tags | Build wheel, build and push Docker image, create Forgejo release with artifacts | + +#### CI Jobs (ci.yml) + +The CI pipeline runs the following jobs. Jobs without explicit dependencies run in parallel: + +| Job | Depends On | Description | +|-----|-----------|-------------| +| `lint` | — | Ruff lint and format check via `nox -s lint` and `nox -s format -- --check` | +| `typecheck` | — | Pyright strict type checking via `nox -s typecheck` | +| `security` | — | Bandit + Semgrep + Vulture via `nox -s security_scan` and `nox -s dead_code` | +| `quality` | — | Radon complexity analysis via `nox -s complexity` | +| `unit_tests` | — | Behave BDD tests via `nox -s unit_tests` | +| `integration_tests` | — | Robot Framework integration tests via `nox -s integration_tests` | +| `e2e_tests` | — | End-to-end Robot tests with real LLM keys via `nox -s e2e_tests` | +| `coverage` | `lint`, `typecheck` | Slipcover coverage report via `nox -s coverage_report` (fail-under 97%) | +| `benchmark-regression` | `lint`, `typecheck` | ASV benchmark regression on PRs via `nox -s benchmark_regression` | +| `benchmark-publish` | — | ASV benchmark publish on push to `master`/`develop` | +| `build` | — | Wheel build via `nox -s build` | +| `docker` | `lint`, `typecheck`, `unit_tests`, `security` | Docker image build and smoke test | +| `status-check` | All required jobs | Consolidation gate for branch protection rules | + +#### Required Checks for Merge + +The `status-check` job is the single gate for branch protection. It depends on all required +jobs and fails if any of them did not succeed. The following jobs must pass for a PR to be +mergeable: + +- **lint** — Code passes Ruff linting and formatting checks +- **typecheck** — Code passes Pyright strict type checking +- **security** — No high-severity Bandit findings, Semgrep rules pass, no dead code +- **unit_tests** — All Behave BDD scenarios pass +- **coverage** — Test coverage >= 97% + +#### How to Read CI Results + +1. Open the PR and check the status checks section at the bottom. +2. Each job appears as a separate check. Click on a failed job to see its logs. +3. The `status-check` job provides a summary of all required job results. +4. For coverage failures, look for the `COVERAGE OK` or `COVERAGE FAILED` summary line in + the coverage job output. +5. Benchmark regression results (on PRs) are uploaded as artifacts for detailed review. + +#### Release Process + +Releases are triggered by pushing a version tag: + +```bash +git tag v3.6.0 +git push origin v3.6.0 +``` + +The `release.yml` workflow then: + +1. **Builds the wheel** via `nox -s build` and uploads it as an artifact. +2. **Builds the Docker image** tagged with the version and `latest`. +3. **Pushes the Docker image** to the container registry (configured via repository secrets). +4. **Creates a Forgejo release** with the wheel attached as a downloadable artifact. + +Required repository secrets for releases: + +| Secret | Purpose | +|--------|---------| +| `CONTAINER_REGISTRY` | Docker registry URL (e.g., `registry.cleverthis.com/cleveragents/core`) | +| `CONTAINER_REGISTRY_USER` | Registry authentication username | +| `CONTAINER_REGISTRY_PASSWORD` | Registry authentication password | +| `FORGEJO_TOKEN` | API token for creating releases | +| `FORGEJO_URL` | Forgejo instance URL (e.g., `https://git.cleverthis.com`) | + +#### Dependency Caching + +CI jobs use `actions/cache@v3` to cache uv package downloads between runs. Cache keys are +based on the `pyproject.toml` hash, so caches are automatically invalidated when dependencies +change. This reduces CI run times by avoiding redundant package downloads. + ### Merge Requirements The general [Review and Merge Requirements](#review-and-merge-requirements) apply to this diff --git a/features/ci_workflow_validation.feature b/features/ci_workflow_validation.feature index 4fe24fddf..eaf486276 100644 --- a/features/ci_workflow_validation.feature +++ b/features/ci_workflow_validation.feature @@ -74,3 +74,83 @@ Feature: CI workflow validation | dead_code | | complexity | | build | + + # --- Release pipeline scenarios --- + + Scenario: Release workflow file exists + Given the CI workflow file at ".forgejo/workflows/release.yml" + Then the CI workflow file should exist + + Scenario: Release workflow YAML is valid + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow YAML should be valid + + Scenario: Release workflow triggers on version tags + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on push tags matching "v*" + + Scenario: Release workflow has build-wheel job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "build-wheel" + And the job "build-wheel" should run "nox -s build" + + Scenario: Release workflow has build-docker job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "build-docker" + + Scenario: Release workflow has create-release job + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "create-release" + + Scenario: Release workflow create-release depends on build jobs + Given the CI workflow file at ".forgejo/workflows/release.yml" + When I parse the CI workflow YAML + Then the job "create-release" should depend on "build-wheel" + And the job "create-release" should depend on "build-docker" + + # --- Status-check consolidation job --- + + Scenario: CI workflow has status-check consolidation job + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should have a job named "status-check" + + Scenario: Status-check job depends on all required jobs + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the job "status-check" should depend on "lint" + And the job "status-check" should depend on "typecheck" + And the job "status-check" should depend on "security" + And the job "status-check" should depend on "unit_tests" + And the job "status-check" should depend on "coverage" + + # --- Coverage threshold --- + + Scenario: CI coverage threshold is 97% + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the coverage job should enforce a 97% threshold + + # --- Branch triggers --- + + Scenario: CI workflow triggers on push to master + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on push to "master" + + Scenario: CI workflow triggers on pull requests to master + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then the workflow should trigger on pull_request to "master" + + # --- Dependency caching --- + + Scenario: CI workflow uses dependency caching + Given the CI workflow file at ".forgejo/workflows/ci.yml" + When I parse the CI workflow YAML + Then at least one job should use actions/cache diff --git a/features/steps/ci_workflow_validation_steps.py b/features/steps/ci_workflow_validation_steps.py index dac3b5e29..49913883f 100644 --- a/features/steps/ci_workflow_validation_steps.py +++ b/features/steps/ci_workflow_validation_steps.py @@ -114,3 +114,78 @@ def step_then_workflow_references_nox_sessions(context): f"Nox session '{session_name}' (as '{nox_call}') " f"not found in any CI workflow step" ) + + +@then("the workflow YAML should be valid") +def step_then_workflow_yaml_valid(context): + """Verify the parsed YAML is a valid workflow (has jobs key).""" + if context.ci_workflow is None: + raise AssertionError("Workflow YAML parsed as None (empty file)") + if "jobs" not in context.ci_workflow: + raise AssertionError("Workflow YAML missing 'jobs' key") + if not isinstance(context.ci_workflow["jobs"], dict): + raise AssertionError("Workflow 'jobs' is not a mapping") + + +@then('the workflow should trigger on push tags matching "{pattern}"') +def step_then_workflow_triggers_on_tags(context, pattern): + """Verify the workflow triggers on push tags matching a pattern.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + push_config = on_config.get("push", {}) + tags = push_config.get("tags", []) + if pattern not in tags: + raise AssertionError(f"Tag pattern '{pattern}' not found in push tags: {tags}") + + +@then("the coverage job should enforce a 97% threshold") +def step_then_coverage_threshold_97(context): + """Verify the coverage job references the 97% threshold.""" + jobs = context.ci_workflow.get("jobs", {}) + coverage_job = jobs.get("coverage") + if coverage_job is None: + raise AssertionError("Coverage job not found in workflow") + + steps = coverage_job.get("steps", []) + all_run_commands = "\n".join(step.get("run", "") for step in steps if "run" in step) + if "97" not in all_run_commands: + raise AssertionError( + "Coverage job does not reference 97% threshold in any step" + ) + + +@then('the workflow should trigger on push to "{branch}"') +def step_then_workflow_triggers_on_push(context, branch): + """Verify the workflow triggers on push to a specific branch.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + push_config = on_config.get("push", {}) + branches = push_config.get("branches", []) + if branch not in branches: + raise AssertionError( + f"Branch '{branch}' not found in push branches: {branches}" + ) + + +@then('the workflow should trigger on pull_request to "{branch}"') +def step_then_workflow_triggers_on_pr(context, branch): + """Verify the workflow triggers on pull_request to a specific branch.""" + on_config = context.ci_workflow.get("on", context.ci_workflow.get(True, {})) + pr_config = on_config.get("pull_request", {}) + branches = pr_config.get("branches", []) + if branch not in branches: + raise AssertionError( + f"Branch '{branch}' not found in pull_request branches: {branches}" + ) + + +@then("at least one job should use actions/cache") +def step_then_at_least_one_job_uses_cache(context): + """Verify at least one job uses actions/cache for dependency caching.""" + jobs = context.ci_workflow.get("jobs", {}) + for job_data in jobs.values(): + for step in job_data.get("steps", []): + uses = step.get("uses", "") + if "actions/cache" in uses: + return + raise AssertionError( + "No job in the CI workflow uses actions/cache for dependency caching" + ) diff --git a/robot/ci_nox_validation.robot b/robot/ci_nox_validation.robot index baa4e420f..6dc2c11f1 100644 --- a/robot/ci_nox_validation.robot +++ b/robot/ci_nox_validation.robot @@ -25,8 +25,62 @@ CI Workflow File Exists [Tags] ci quality slow File Should Exist .forgejo/workflows/ci.yml +Release Workflow File Exists + [Documentation] Verify the release workflow file is present in the repository + [Tags] ci quality slow + File Should Exist .forgejo/workflows/release.yml + +Nightly Quality Workflow File Exists + [Documentation] Verify the nightly quality workflow file is present + [Tags] ci quality slow + File Should Exist .forgejo/workflows/nightly-quality.yml + Nox Lint Session Runs Successfully [Documentation] Verify that nox lint session can be invoked [Tags] ci quality slow ${result}= Run Process nox -s lint timeout=120s on_timeout=kill Should Be Equal As Integers ${result.rc} 0 msg=nox -s lint failed: ${result.stderr} + +Release Workflow Contains Build Wheel Job + [Documentation] Verify the release workflow defines a build-wheel job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} build-wheel: + +Release Workflow Contains Build Docker Job + [Documentation] Verify the release workflow defines a build-docker job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} build-docker: + +Release Workflow Contains Create Release Job + [Documentation] Verify the release workflow defines a create-release job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} create-release: + +Release Workflow Triggers On Version Tags + [Documentation] Verify the release workflow triggers on v* tags + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} tags: + Should Contain ${content} "v*" + +CI Workflow Has Status Check Job + [Documentation] Verify the CI workflow has a status-check consolidation job + [Tags] ci quality + ${content}= Get File .forgejo/workflows/ci.yml + Should Contain ${content} status-check: + +CI Workflow Uses Dependency Caching + [Documentation] Verify the CI workflow uses actions/cache for uv packages + [Tags] ci quality + ${content}= Get File .forgejo/workflows/ci.yml + Should Contain ${content} actions/cache@v3 + Should Contain ${content} ~/.cache/uv + +Nox Build Session Is Referenced In Release Workflow + [Documentation] Verify the release workflow uses nox -s build + [Tags] ci quality + ${content}= Get File .forgejo/workflows/release.yml + Should Contain ${content} nox -s build