From a1fd36932fa19e50520919bbe5d5693b5b75fad7 Mon Sep 17 00:00:00 2001 From: "Luis Mendes (CoreRasurae)" Date: Fri, 31 Jul 2026 11:55:37 +0000 Subject: [PATCH] fix(ci): make benchmark_regression job fast and correctly fail on regressions Three compounding issues in the benchmark_regression CI check, found while investigating a 41m13s hang/timeout on PR #85's run (job 205876): 1. `asv continuous` ran with no speed-oriented flags, fully calibrating and repeating every one of 138 benchmark runs (69 benchmarks x 2 commits). Per-benchmark ASV overhead (calibration + repeat + process averaging) cost 20-90s of wall clock per benchmark, dwarfing the actual measured microsecond-to-millisecond costs. `noxfile.py`'s `benchmark_regression` session now forwards *session.posargs to `asv continuous`, so its own default (full, accurate) behavior is unchanged when called without extra args; `.gitea/workflows/ci.yml`'s `benchmark` job now invokes `nox -s benchmark_regression -- --quick`, scoping ASV's single-run mode to this informational job only. 2. With --quick applied, a new bottleneck surfaced: asv.conf.json's build/install/uninstall commands used plain `python -m build`/`pip`, which silently spent ~11 minutes per run in pip's resolver (build isolation + a full, uncached dependency install per compared commit, no progress output during backtracking). Switched to `uv build`/ `uv pip install`/`uv pip uninstall`. `uv` isn't installed inside each ASV-managed virtualenv by default (asv's find_executable only searches that venv's own bin/, not $PATH), so added `uv` to asv's `matrix` config to have it pip-installed during venv bootstrap (already-fast for a single small package), and pass `--python {env_dir}/bin/python` explicitly to each command since standalone `uv` (unlike `python -m pip`) has no unambiguous way to infer which of asv's several per-commit venvs to target otherwise. Also added `--force-reinstall` to the install command per asv's own documented rationale (compared commits may share the same package version). 3. `asv continuous` returns the boolean `worsened` as its process exit code (0 = no significant regression, 1 = a benchmark measurably worsened past --factor) -- there is no exit code 2 in this asv version. `noxfile.py`'s `success_codes=[0, 2]` therefore never actually matched the real "worsened" code, silently making this session fail on *any* regression signal regardless of significance. Corrected to `success_codes=[0]`, so the session (and CI job step) now fails specifically when asv reports a real, significant regression -- the CI job's existing `continue-on-error: true` is what keeps this from blocking the PR, not this success list. Verified locally end-to-end (rm -rf .asv/env .asv/results between runs): completes in ~3-11 minutes (vs. the prior 40+ minute hang/timeout), and the session now correctly reports "failed" when asv detects a significant regression between the compared commits. ISSUES CLOSED: #86 --- .gitea/workflows/ci.yml | 7 ++++++- asv.conf.json | 9 ++++++--- noxfile.py | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 56555b6..14364c7 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -389,7 +389,12 @@ jobs: - name: Run ASV benchmark regression check via nox run: | mkdir -p build - nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-output.log + # --quick: skip per-benchmark calibration/repeat/process + # averaging (asv default). This is a fast, informational + # trend check (continue-on-error), not the accurate, + # fully-calibrated run used for real performance + # investigation (nox -s benchmark, no --quick). + nox -s benchmark_regression -- --quick 2>&1 | tee build/nox-benchmark-output.log - name: Upload benchmark log artifact if: always() diff --git a/asv.conf.json b/asv.conf.json index 1e4b29f..d84f4a5 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -11,13 +11,16 @@ "env_dir": ".asv/env", "results_dir": ".asv/results", "html_dir": ".asv/html", + "matrix": { + "uv": [""] + }, "build_command": [ - "python -m build --wheel -o {build_cache_dir} {build_dir}" + "uv build --python {env_dir}/bin/python --wheel -o {build_cache_dir} {build_dir}" ], "install_command": [ - "python -mpip install {wheel_file}" + "uv pip install --python {env_dir}/bin/python {wheel_file} --force-reinstall" ], "uninstall_command": [ - "return-code=any python -mpip uninstall -y {project}" + "return-code=any uv pip uninstall --python {env_dir}/bin/python {project}" ] } diff --git a/noxfile.py b/noxfile.py index 78db5b2..91c320c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -364,7 +364,21 @@ def benchmark_regression(session: nox.Session) -> None: base = os.environ.get("BENCHMARK_BASE", "origin/master") session.run("asv", "machine", "--yes") session.run( - "asv", "continuous", base, "HEAD", "--show-stderr", success_codes=[0, 2] + "asv", + "continuous", + base, + "HEAD", + "--show-stderr", + *session.posargs, + # asv continuous returns the boolean `worsened` as its exit code: + # 0 = no significant regression, 1 = a benchmark measurably + # worsened past --factor. This session should fail on 1 so the + # regression is visible (the CI job's continue-on-error: true is + # what keeps this from blocking the PR, not this success list). + # There is no exit code 2 in this asv version; the prior [0, 2] + # never matched the real "worsened" code and made this session + # fail unconditionally on any regression signal. + success_codes=[0], ) -- 2.52.0