fix(ci): make benchmark_regression job fast and correctly fail on regressions #87

Open
CoreRasurae wants to merge 1 commits from task/m2-quick-benchmark-regression-ci into master
3 changed files with 27 additions and 5 deletions
+6 -1
View File
@@ -389,7 +389,12 @@ jobs:
- name: Run ASV benchmark regression check via nox
run: |
mkdir -p build
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-output.log
# --quick: skip per-benchmark calibration/repeat/process
# averaging (asv default). This is a fast, informational
# trend check (continue-on-error), not the accurate,
# fully-calibrated run used for real performance
# investigation (nox -s benchmark, no --quick).
nox -s benchmark_regression -- --quick 2>&1 | tee build/nox-benchmark-output.log
- name: Upload benchmark log artifact
if: always()
+6 -3
View File
@@ -11,13 +11,16 @@
"env_dir": ".asv/env",
"results_dir": ".asv/results",
"html_dir": ".asv/html",
"matrix": {
"uv": [""]
},
"build_command": [
"python -m build --wheel -o {build_cache_dir} {build_dir}"
"uv build --python {env_dir}/bin/python --wheel -o {build_cache_dir} {build_dir}"
],
"install_command": [
"python -mpip install {wheel_file}"
"uv pip install --python {env_dir}/bin/python {wheel_file} --force-reinstall"
],
"uninstall_command": [
"return-code=any python -mpip uninstall -y {project}"
"return-code=any uv pip uninstall --python {env_dir}/bin/python {project}"
]
}
+15 -1
View File
@@ -364,7 +364,21 @@ def benchmark_regression(session: nox.Session) -> None:
base = os.environ.get("BENCHMARK_BASE", "origin/master")
session.run("asv", "machine", "--yes")
session.run(
"asv", "continuous", base, "HEAD", "--show-stderr", success_codes=[0, 2]
"asv",
"continuous",
base,
"HEAD",
"--show-stderr",
*session.posargs,
# asv continuous returns the boolean `worsened` as its exit code:
# 0 = no significant regression, 1 = a benchmark measurably
# worsened past --factor. This session should fail on 1 so the
# regression is visible (the CI job's continue-on-error: true is
# what keeps this from blocking the PR, not this success list).
# There is no exit code 2 in this asv version; the prior [0, 2]
# never matched the real "worsened" code and made this session
# fail unconditionally on any regression signal.
success_codes=[0],
)