diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 0d5388e64..75cee7444 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -207,6 +207,116 @@ jobs: build/htmlcov/ retention-days: 30 + benchmark-regression: + if: forgejo.event_name == 'pull_request' + runs-on: docker + container: + image: python:3.13-slim + needs: [lint, typecheck] + steps: + - name: Install system dependencies (nodejs for checkout, git for merge tests) + run: | + apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + + - name: Checkout full history + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Compute base commit + id: hash + run: | + git fetch origin "${{ forgejo.base_ref }}" --depth=200 + BASE_SHA=$(git merge-base HEAD "origin/${{ forgejo.base_ref }}") + echo "ASV_BASE_SHA=${BASE_SHA}" >> $FORGEJO_OUTPUT + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox + + - name: Restore prior ASV benchmarks + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }} + run: | + python -m pip install awscli + mkdir -p build/asv/results + aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true + + - name: Run asv continuous via nox + env: + ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }} + run: | + nox -s benchmark_regression + + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: asv-results-pr + path: | + build/asv/results + build/asv/html + rentention-days: 30 + + benchmark-publish: + if: forgejo.event_name == 'push' && ( forgejo.ref == 'refs/heads/master' || forgejo.ref == 'refs/heads/develop' ) + runs-on: docker + container: python:3.13-slim + steps: + - name: Install system dependencies (nodejs for checkout, git for merge tests) + run: | + apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + + - name: Checkout full history + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox + + - name: Restore prior ASV benchmarks + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }} + run: | + python -m pip install awscli + mkdir -p build/asv/results + aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true + + - name: Run asv via nox + run: | + nox -s benchmark + + - name: Upload updated benchmarks and website to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }} + run: | + python -m pip install awscli + aws s3 sync build/asv/results "s3://${ASV_S3_BUCKET}/asv/results" --delete + aws s3 sync build/asv/html "s3://${ASV_S3_BUCKET}/asv/html" --delete + + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: asv-results + path: | + build/asv/results + build/asv/html + rentention-days: 0 + build: runs-on: docker container: diff --git a/noxfile.py b/noxfile.py index 3e25b468c..b2b158cb0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -688,10 +688,39 @@ def benchmark(session: nox.Session): session.install("-e", ".[tests]") config_path = "asv.conf.json" session.run("asv", "machine", "--yes", f"--config={config_path}") - session.run("asv", "run", "--show-stderr", "--verbose", f"--config={config_path}") + session.run( + "asv", + "run", + "--append-samples", + "--show-stderr", + "--verbose", + f"--config={config_path}", + success_codes=[0, 2], + ) session.run("asv", "publish", f"--config={config_path}") +@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") +def benchmark_regression(session: nox.Session): + """Run Airspeed Velocity benchmarks regression test.""" + session.install("-e", ".[tests]") + config_path = "asv.conf.json" + asv_base_sha = os.environ.get("ASV_BASE_SHA") + session.run("asv", "machine", "--yes", f"--config={config_path}") + session.run( + "asv", + "continuous", + "--append-samples", + "--show-stderr", + "--verbose", + f"--factor=1.10", + f"--config={config_path}", + asv_base_sha, + "HEAD", + success_codes=[0, 2], + ) + + # Sessions to run by default when running `nox` without arguments nox.options.sessions = [ "lint", # ~5-10 seconds