diff --git a/.forgejo/workflows/master.yml b/.forgejo/workflows/master.yml index 679165383..9c8c1f557 100644 --- a/.forgejo/workflows/master.yml +++ b/.forgejo/workflows/master.yml @@ -3,8 +3,6 @@ name: CI on: push: branches: [master, develop] - pull_request: - branches: [master, develop] env: UV_VERSION: "0.8.0" @@ -15,6 +13,7 @@ jobs: benchmark-regression: if: forgejo.event_name == 'pull_request' runs-on: docker-benchmark + timeout-minutes: 35 container: image: python:3.13-slim @@ -51,11 +50,11 @@ jobs: mkdir -p build/asv/results aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true - - name: Run asv continuous via nox + - name: Run fast benchmark regression via nox env: ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }} run: | - nox -s benchmark_regression + nox -s benchmark_regression_fast - name: Archive the results run: | diff --git a/.forgejo/workflows/nightly-quality.yml b/.forgejo/workflows/nightly-quality.yml index 372c6dd9a..1977c7fd5 100644 --- a/.forgejo/workflows/nightly-quality.yml +++ b/.forgejo/workflows/nightly-quality.yml @@ -18,11 +18,13 @@ jobs: container: image: python:3.13-slim steps: - - name: Install system dependencies (git for merge tests) + - name: Install system dependencies (git for merge tests and ASV) run: | apt-get update && apt-get install -y -qq git && rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install uv run: | @@ -108,6 +110,14 @@ jobs: print(json.dumps(report, indent=2)) " + - name: Run full benchmark regression via nox + run: | + nox -s benchmark_regression + env: + NOX_DEFAULT_VENV_BACKEND: uv + # Nightly run always compares against master HEAD. + ASV_BASE_SHA: master + - name: Upload quality reports if: always() uses: actions/upload-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bd4bb64..352e44fb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). logged at debug level for observability. ### Added +- **Benchmark Regression CI Optimization** (#1668): Added `benchmark_regression_fast` + nox session that excludes the three slowest benchmark suites + (`IndexingScalingSuite`, `ContextAssemblyScalingSuite`, `ExecutionThroughputSuite`) + from PR regression checks. Added `benchmark_regression` CI job to `ci.yml` using + the fast session with a 20-minute timeout. Added full `benchmark_regression` run + to the nightly quality workflow. Documented the excluded suites and their timeout + characteristics in each benchmark file. + - Wired `StrategyActor` into the real plan execution path: `_get_plan_executor` in `plan.py` now resolves the strategy actor via `resolve_strategy_actor()` (reading the `actor.default.strategy` config key) instead of always diff --git a/benchmarks/context_assembly_scaling_bench.py b/benchmarks/context_assembly_scaling_bench.py index e5fea8559..ce9192f51 100644 --- a/benchmarks/context_assembly_scaling_bench.py +++ b/benchmarks/context_assembly_scaling_bench.py @@ -60,7 +60,12 @@ def _make_fragments(count: int) -> list[ContextFragment]: class ContextAssemblyScalingSuite: - """Benchmark ACMS context assembly at production fragment counts.""" + """Benchmark ACMS context assembly at production fragment counts. + + Full parameter set: [100, 1_000, 5_000, 10_000] (used in nightly runs). + This suite is excluded from the fast PR subset via ``benchmark_regression_fast`` + because the 5K and 10K cases require up to 300 s each. See issue #1668. + """ params: ClassVar[list[int]] = [100, 1_000, 5_000, 10_000] param_names: ClassVar[list[str]] = ["fragment_count"] diff --git a/benchmarks/execution_throughput_bench.py b/benchmarks/execution_throughput_bench.py index b88540972..971cfc761 100644 --- a/benchmarks/execution_throughput_bench.py +++ b/benchmarks/execution_throughput_bench.py @@ -81,7 +81,12 @@ def _execute_single_plan(runner: ToolRunner) -> None: class ExecutionThroughputSuite: - """Benchmark plan execution throughput at varying plan counts.""" + """Benchmark plan execution throughput at varying plan counts. + + Full parameter set: [10, 50, 100] (used in nightly runs). + This suite is excluded from the fast PR subset via ``benchmark_regression_fast`` + because the 50 and 100 plan cases require up to 300 s each. See issue #1668. + """ params: ClassVar[list[int]] = [10, 50, 100] param_names: ClassVar[list[str]] = ["plan_count"] diff --git a/benchmarks/large_project_scaling_bench.py b/benchmarks/large_project_scaling_bench.py index 67cbc595a..860606eef 100644 --- a/benchmarks/large_project_scaling_bench.py +++ b/benchmarks/large_project_scaling_bench.py @@ -98,7 +98,12 @@ _DEFAULT_EXCLUDE: tuple[str, ...] = ("*.pyc", "__pycache__/*") class IndexingScalingSuite: - """Benchmark ``walk_and_index`` at production-scale file counts.""" + """Benchmark ``walk_and_index`` at production-scale file counts. + + Full parameter set: [1_000, 10_000, 50_000, 100_000] (used in nightly runs). + This suite is excluded from the fast PR subset via ``benchmark_regression_fast`` + because the 50K and 100K cases require up to 600 s each. See issue #1668. + """ params: ClassVar[list[int]] = [1_000, 10_000, 50_000, 100_000] param_names: ClassVar[list[str]] = ["file_count"] diff --git a/noxfile.py b/noxfile.py index 957467855..50c84e6bb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -842,7 +842,14 @@ def benchmark(session: nox.Session): @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def benchmark_regression(session: nox.Session): - """Run Airspeed Velocity benchmarks regression test.""" + """Run Airspeed Velocity benchmarks regression test (full suite). + + Runs the complete benchmark suite comparing HEAD against the base SHA. + This is the authoritative regression check intended for nightly CI + runs where wall-clock time is not a constraint. + + Set ``ASV_BASE_SHA`` to override the comparison base (default: master). + """ session.install("-e", ".[tests]") config_path = "asv.conf.json" asv_base_sha = os.environ.get("ASV_BASE_SHA", "master") @@ -873,6 +880,71 @@ def benchmark_regression(session: nox.Session): session.run("asv", "publish", f"--config={config_path}") +@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") +def benchmark_regression_fast(session: nox.Session): + """Run a fast subset of ASV benchmarks for PR regression checks. + + Excludes the three slowest benchmark suites identified in the CI + execution-time analysis (see issue #1668): + + 1. ``IndexingScalingSuite`` (large_project_scaling_bench) -- 600 s timeout, + runs ``walk_and_index`` at up to 100 K files. + + 2. ``ContextAssemblyScalingSuite`` (context_assembly_scaling_bench) -- + 300 s timeout, assembles ACMS context at up to 10 K fragments. + + 3. ``ExecutionThroughputSuite`` (execution_throughput_bench) -- 300 s + timeout, executes up to 100 sequential plans. + + The full suite is still run nightly via ``benchmark_regression``. + + Set ``ASV_BASE_SHA`` to override the comparison base (default: master). + Target wall-clock time: under 15 minutes on a standard CI runner. + """ + session.install("-e", ".[tests]") + config_path = "asv.conf.json" + asv_base_sha = os.environ.get("ASV_BASE_SHA", "master") + + # Regex pattern that excludes the three slowest suites. + # ASV --bench accepts a Python regex matched against + # "..". + fast_bench_pattern = ( + "^(?!.*(" + "IndexingScalingSuite" + "|ContextAssemblyScalingSuite" + "|ExecutionThroughputSuite" + ")).*" + ) + + session.run( + "asv", + "machine", + "--machine=forgejo-runner", + "--os=Linux_6.x", + "--arch=x86_64", + "--num_cpu=32", + "--ram=32GB", + "--cpu=AMD", + f"--config={config_path}", + ) + session.run( + "asv", + "continuous", + "--machine=forgejo-runner", + "--append-samples", + "--show-stderr", + "--verbose", + "--factor=1.50", + "--bench", + fast_bench_pattern, + f"--config={config_path}", + asv_base_sha, + "HEAD", + success_codes=[0, 2], + ) + session.run("asv", "publish", f"--config={config_path}") + + # Sessions to run by default when running `nox` without arguments nox.options.sessions = [ "lint", # ~5-10 seconds