fix(ci): make benchmark_regression job fast and correctly fail on regressions #87
@@ -389,7 +389,12 @@ jobs:
|
||||
- name: Run ASV benchmark regression check via nox
|
||||
run: |
|
||||
mkdir -p build
|
||||
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-output.log
|
||||
# --quick: skip per-benchmark calibration/repeat/process
|
||||
# averaging (asv default). This is a fast, informational
|
||||
# trend check (continue-on-error), not the accurate,
|
||||
# fully-calibrated run used for real performance
|
||||
# investigation (nox -s benchmark, no --quick).
|
||||
nox -s benchmark_regression -- --quick 2>&1 | tee build/nox-benchmark-output.log
|
||||
|
||||
- name: Upload benchmark log artifact
|
||||
if: always()
|
||||
|
||||
+6
-3
@@ -11,13 +11,16 @@
|
||||
"env_dir": ".asv/env",
|
||||
"results_dir": ".asv/results",
|
||||
"html_dir": ".asv/html",
|
||||
"matrix": {
|
||||
"uv": [""]
|
||||
},
|
||||
"build_command": [
|
||||
"python -m build --wheel -o {build_cache_dir} {build_dir}"
|
||||
"uv build --python {env_dir}/bin/python --wheel -o {build_cache_dir} {build_dir}"
|
||||
],
|
||||
"install_command": [
|
||||
"python -mpip install {wheel_file}"
|
||||
"uv pip install --python {env_dir}/bin/python {wheel_file} --force-reinstall"
|
||||
],
|
||||
"uninstall_command": [
|
||||
"return-code=any python -mpip uninstall -y {project}"
|
||||
"return-code=any uv pip uninstall --python {env_dir}/bin/python {project}"
|
||||
]
|
||||
}
|
||||
|
||||
+15
-1
@@ -364,7 +364,21 @@ def benchmark_regression(session: nox.Session) -> None:
|
||||
base = os.environ.get("BENCHMARK_BASE", "origin/master")
|
||||
session.run("asv", "machine", "--yes")
|
||||
session.run(
|
||||
"asv", "continuous", base, "HEAD", "--show-stderr", success_codes=[0, 2]
|
||||
"asv",
|
||||
"continuous",
|
||||
base,
|
||||
"HEAD",
|
||||
"--show-stderr",
|
||||
*session.posargs,
|
||||
# asv continuous returns the boolean `worsened` as its exit code:
|
||||
# 0 = no significant regression, 1 = a benchmark measurably
|
||||
# worsened past --factor. This session should fail on 1 so the
|
||||
# regression is visible (the CI job's continue-on-error: true is
|
||||
# what keeps this from blocking the PR, not this success list).
|
||||
# There is no exit code 2 in this asv version; the prior [0, 2]
|
||||
# never matched the real "worsened" code and made this session
|
||||
# fail unconditionally on any regression signal.
|
||||
success_codes=[0],
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user