clever-agent
81229422f2
CI / status-check (pull_request) Blocked by required conditions
CI / lint (pull_request) Successful in 52s
CI / quality (pull_request) Successful in 1m13s
CI / typecheck (pull_request) Successful in 1m20s
CI / security (pull_request) Successful in 1m41s
CI / build (pull_request) Successful in 30s
CI / e2e_tests (pull_request) Failing after 4m9s
CI / integration_tests (pull_request) Successful in 4m57s
CI / unit_tests (pull_request) Successful in 6m1s
CI / helm (pull_request) Successful in 27s
CI / push-validation (pull_request) Successful in 22s
CI / coverage (pull_request) Has started running
CI / docker (pull_request) Successful in 1m31s
This reverts commit 9c5f19854d.
193 lines
7.8 KiB
YAML
193 lines
7.8 KiB
YAML
name: Benchmark Regression
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 2 * * *"
|
|
- cron: "0 3 * * 0"
|
|
workflow_dispatch:
|
|
inputs:
|
|
base_sha:
|
|
description: "Base SHA or branch to compare against (default: master)"
|
|
required: false
|
|
default: "master"
|
|
run_full_suite:
|
|
description: "Run full benchmark suite (true) or regression only (false)"
|
|
required: false
|
|
default: "false"
|
|
|
|
env:
|
|
UV_VERSION: "0.8.0"
|
|
PYTHON_VERSION: "3.13"
|
|
NOX_DEFAULT_VENV_BACKEND: "uv"
|
|
|
|
jobs:
|
|
benchmark-regression:
|
|
if: github.event_name == 'schedule' && github.event.schedule == '0 2 * * *' || github.event_name == 'workflow_dispatch' && github.event.inputs.run_full_suite == 'false'
|
|
runs-on: docker
|
|
timeout-minutes: 120
|
|
container:
|
|
image: python:3.13-slim
|
|
steps:
|
|
- name: Install system dependencies
|
|
run: |
|
|
apt-get update && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/*
|
|
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install uv and nox
|
|
run: |
|
|
pip install -q uv==${{ env.UV_VERSION }} nox
|
|
|
|
- name: Cache uv packages
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: ~/.cache/uv
|
|
key: uv-benchmark-${{ hashFiles('pyproject.toml') }}
|
|
restore-keys: |
|
|
uv-benchmark-
|
|
uv-
|
|
|
|
- name: Sync benchmark results from S3
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
|
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
|
run: |
|
|
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
|
pip install -q awscli
|
|
mkdir -p build/asv/results
|
|
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results/" build/asv/results/ || echo "No existing results to sync"
|
|
else
|
|
echo "Skipping S3 sync - AWS credentials not configured"
|
|
fi
|
|
|
|
- name: Run benchmark regression via nox
|
|
env:
|
|
NOX_DEFAULT_VENV_BACKEND: uv
|
|
ASV_BASE_SHA: ${{ github.event.inputs.base_sha || 'master' }}
|
|
run: |
|
|
mkdir -p build
|
|
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-regression-output.log
|
|
|
|
- name: Publish benchmark results to S3
|
|
if: always()
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
|
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
|
run: |
|
|
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
|
pip install -q awscli
|
|
aws s3 sync build/asv/results/ "s3://${ASV_S3_BUCKET}/asv/results/" || echo "Failed to publish results to S3"
|
|
aws s3 sync build/asv/html/ "s3://${ASV_S3_BUCKET}/asv/html/" || echo "Failed to publish HTML to S3"
|
|
else
|
|
echo "Skipping S3 publish - AWS credentials not configured"
|
|
fi
|
|
|
|
- name: Upload benchmark log artifact
|
|
if: always()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: benchmark-regression-logs
|
|
path: build/nox-benchmark-regression-output.log
|
|
retention-days: 30
|
|
|
|
- name: Upload benchmark results artifact
|
|
if: always()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: benchmark-regression-results
|
|
path: |
|
|
build/asv/results/
|
|
build/asv/html/
|
|
retention-days: 90
|
|
|
|
benchmark-full:
|
|
if: github.event_name == 'schedule' && github.event.schedule == '0 3 * * 0' || github.event_name == 'workflow_dispatch' && github.event.inputs.run_full_suite == 'true'
|
|
runs-on: docker
|
|
timeout-minutes: 180
|
|
container:
|
|
image: python:3.13-slim
|
|
steps:
|
|
- name: Install system dependencies
|
|
run: |
|
|
apt-get update && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/*
|
|
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install uv and nox
|
|
run: |
|
|
pip install -q uv==${{ env.UV_VERSION }} nox
|
|
|
|
- name: Cache uv packages
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: ~/.cache/uv
|
|
key: uv-benchmark-full-${{ hashFiles('pyproject.toml') }}
|
|
restore-keys: |
|
|
uv-benchmark-full-
|
|
uv-benchmark-
|
|
uv-
|
|
|
|
- name: Sync benchmark results from S3
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
|
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
|
run: |
|
|
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
|
pip install -q awscli
|
|
mkdir -p build/asv/results
|
|
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results/" build/asv/results/ || echo "No existing results to sync"
|
|
else
|
|
echo "Skipping S3 sync - AWS credentials not configured"
|
|
fi
|
|
|
|
- name: Run full benchmark suite via nox
|
|
env:
|
|
NOX_DEFAULT_VENV_BACKEND: uv
|
|
run: |
|
|
mkdir -p build
|
|
nox -s benchmark 2>&1 | tee build/nox-benchmark-full-output.log
|
|
|
|
- name: Publish benchmark results to S3
|
|
if: always()
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
|
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
|
run: |
|
|
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
|
pip install -q awscli
|
|
aws s3 sync build/asv/results/ "s3://${ASV_S3_BUCKET}/asv/results/" || echo "Failed to publish results to S3"
|
|
aws s3 sync build/asv/html/ "s3://${ASV_S3_BUCKET}/asv/html/" || echo "Failed to publish HTML to S3"
|
|
else
|
|
echo "Skipping S3 publish - AWS credentials not configured"
|
|
fi
|
|
|
|
- name: Upload benchmark log artifact
|
|
if: always()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: benchmark-full-logs
|
|
path: build/nox-benchmark-full-output.log
|
|
retention-days: 30
|
|
|
|
- name: Upload benchmark results artifact
|
|
if: always()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: benchmark-full-results
|
|
path: |
|
|
build/asv/results/
|
|
build/asv/html/
|
|
retention-days: 90
|