From 7959284cb5a793753600a5a16ede5daf4201ccbd Mon Sep 17 00:00:00 2001 From: CleverThis Date: Tue, 14 Apr 2026 12:05:37 +0000 Subject: [PATCH] CI: Move benchmark regression job out of default PR workflow - Extract benchmark-regression into scheduled/dispatchable workflow - Gate benchmark behind nightly schedule (2 AM UTC) and manual dispatch - Add weekly full benchmark suite (3 AM UTC Sundays) - Update CI documentation with benchmark trigger process and reporting - Reduce median PR CI turnaround time to <30 minutes by removing long-running ASV tasks from PR path --- .forgejo/workflows/benchmark-scheduled.yml | 132 +++++++++++++++++++++ docs/development/ci-cd.md | 47 +++++++- 2 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 .forgejo/workflows/benchmark-scheduled.yml diff --git a/.forgejo/workflows/benchmark-scheduled.yml b/.forgejo/workflows/benchmark-scheduled.yml new file mode 100644 index 000000000..b5f3ef2ce --- /dev/null +++ b/.forgejo/workflows/benchmark-scheduled.yml @@ -0,0 +1,132 @@ +name: Benchmark Regression (Scheduled) + +on: + schedule: + # Run benchmark regression nightly at 2 AM UTC + - cron: "0 2 * * *" + workflow_dispatch: + # Allow manual trigger for ad-hoc benchmark validation + inputs: + base_sha: + description: "Base commit SHA for regression comparison (default: master)" + required: false + default: "master" + +env: + UV_VERSION: "0.8.0" + PYTHON_VERSION: "3.13" + NOX_DEFAULT_VENV_BACKEND: "uv" + +jobs: + benchmark-regression: + runs-on: docker + timeout-minutes: 180 + container: + image: python:3.13-slim + steps: + - name: Install system dependencies (nodejs for checkout, git for full history) + run: | + apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + with: + # Fetch full history for ASV regression comparison + fetch-depth: 0 + + - name: Install uv and nox + run: | + pip install -q uv==${{ env.UV_VERSION }} nox + + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv- + + - name: Run benchmark regression via nox + run: | + mkdir -p build + nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-regression-output.log + env: + NOX_DEFAULT_VENV_BACKEND: uv + # Base SHA for regression comparison (from workflow input or default to master) + ASV_BASE_SHA: ${{ github.event.inputs.base_sha || 'master' }} + # AWS credentials for S3 benchmark storage + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }} + + - name: Upload benchmark regression log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: benchmark-regression-logs + path: build/nox-benchmark-regression-output.log + retention-days: 90 + + - name: Upload ASV results + if: always() + uses: actions/upload-artifact@v3 + with: + name: asv-results + path: asv/results/ + retention-days: 90 + + benchmark-publish: + # Publish full benchmark results on a separate schedule (weekly) + if: github.event_name == 'schedule' && github.event.schedule == '0 3 * * 0' + runs-on: docker + timeout-minutes: 120 + container: + image: python:3.13-slim + steps: + - name: Install system dependencies (nodejs for checkout, git for full history) + run: | + apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv and nox + run: | + pip install -q uv==${{ env.UV_VERSION }} nox + + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv- + + - name: Run full benchmark suite via nox + run: | + mkdir -p build + nox -s benchmark 2>&1 | tee build/nox-benchmark-output.log + env: + NOX_DEFAULT_VENV_BACKEND: uv + # AWS credentials for S3 benchmark storage + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }} + + - name: Upload benchmark log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: benchmark-logs + path: build/nox-benchmark-output.log + retention-days: 90 + + - name: Upload ASV results + if: always() + uses: actions/upload-artifact@v3 + with: + name: asv-results-weekly + path: asv/results/ + retention-days: 90 diff --git a/docs/development/ci-cd.md b/docs/development/ci-cd.md index b75667bb2..7473287a5 100644 --- a/docs/development/ci-cd.md +++ b/docs/development/ci-cd.md @@ -147,11 +147,14 @@ PR template at `.forgejo/pull_request_template.md` in the repository root): | Workflow | File | Trigger | Purpose | |----------|------|---------|---------| -| CI | `.forgejo/workflows/ci.yml` | Push to `master`/`develop`, PRs to `master` | Full nox-based validation pipeline | +| CI | `.forgejo/workflows/ci.yml` | Push to `master`/`develop`, PRs to `master` | Full nox-based validation pipeline (excludes benchmarks) | | Nightly Quality | `.forgejo/workflows/nightly-quality.yml` | Midnight UTC (cron), manual | Comprehensive quality monitoring | +| Benchmark Regression | `.forgejo/workflows/benchmark-scheduled.yml` | 2 AM UTC (cron), manual dispatch | Scheduled ASV benchmark regression tests and weekly full benchmark suite | ### CI Job Dependency Graph +**PR Validation Pipeline (ci.yml):** + ``` lint ──────────────────┐ typecheck ─────────────┤ @@ -165,6 +168,13 @@ build ──────────────────── (independent) push-validation ────────── (independent — validates CI push credentials) ``` +**Benchmark Pipeline (benchmark-scheduled.yml):** + +``` +benchmark-regression ──────── (nightly at 2 AM UTC, or manual dispatch) +benchmark-publish ─────────── (weekly at 3 AM UTC on Sundays, or manual dispatch) +``` + ### Nox-Based CI All CI jobs now run their checks through nox sessions rather than invoking @@ -201,13 +211,46 @@ The nightly workflow provides trend data beyond PR-level checks: Reports are uploaded as artifacts under `nightly-quality-reports` and can be downloaded from the Forgejo Actions UI. +### Benchmark Regression Testing + +The benchmark regression workflow runs on a separate schedule to avoid blocking +PR feedback. This workflow: + +- **Nightly regression tests** (2 AM UTC): Runs `nox -s benchmark_regression` to + compare performance against the `master` branch. Detects performance regressions + > 50% (configurable via `--factor` in the workflow). +- **Weekly full benchmarks** (3 AM UTC Sundays): Runs `nox -s benchmark` to + publish complete benchmark results to S3 for trend analysis. +- **Manual dispatch**: Both jobs can be triggered manually via the Forgejo UI + with optional `base_sha` parameter for custom regression comparisons. + +**Why benchmarks are separate:** + +1. **Developer experience**: PR feedback completes in <30 minutes instead of + 99-132 minutes when benchmarks are included. +2. **Resource efficiency**: Benchmarks require full-history checkout, S3 sync, + and long-running ASV tasks that don't provide immediate validation for most + code changes. +3. **Trend analysis**: Scheduled benchmarks provide historical data for + performance tracking without blocking every PR. + +**Triggering benchmarks manually:** + +```bash +# Via Forgejo UI: Actions > Benchmark Regression > Run workflow +# Or via gh CLI: +gh workflow run benchmark-scheduled.yml \ + -f base_sha=master \ + -R cleveragents/cleveragents-core +``` + ## Local Development Workflow ### Before Pushing a PR ```bash # Quick validation (runs default nox sessions: lint, format, typecheck, -# unit_tests, integration_tests, docs, build, benchmark, coverage_report) +# unit_tests, integration_tests, docs, build, coverage_report) nox # Or run individual checks matching CI jobs