From e77cf8bb1c5dfed905bd707fa44e4e882606fdb3 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Mon, 16 Mar 2026 22:27:08 +0000 Subject: [PATCH 1/2] feat(perf): large project scaling tests Add ASV benchmark suites for large project scaling: indexing at 10K/50K/100K files with walk_and_index throughput and incremental refresh metrics, context assembly at 1K/5K/10K fragments, and execution throughput for sequential/concurrent plans. Update scale fixture thresholds for 50K/100K profiles. Add Behave and Robot tests for baseline validation. Document scaling baselines. Also fixes pre-existing test failures and integrates recent master changes discovered during full nox suite verification. ISSUES CLOSED: #859 --- CHANGELOG.md | 4 + benchmarks/bench_uko_layer3.py | 10 +- benchmarks/context_assembly_scaling_bench.py | 138 +++++++++++ benchmarks/execution_throughput_bench.py | 123 ++++++++++ benchmarks/large_project_scaling_bench.py | 181 +++++++++++++++ docs/reference/scaling_baselines.md | 105 +++++++++ features/consolidated_misc.feature | 8 +- .../fixtures/scale/baseline_thresholds.json | 84 +++++++ features/fixtures/scale/scale_metadata.json | 33 +++ features/scaling_performance.feature | 100 ++++++++ features/steps/scaling_performance_steps.py | 216 ++++++++++++++++++ robot/helper_scale_test.py | 10 +- robot/helper_scaling_performance.py | 207 +++++++++++++++++ robot/scaling_performance.robot | 60 +++++ 14 files changed, 1270 insertions(+), 9 deletions(-) create mode 100644 benchmarks/context_assembly_scaling_bench.py create mode 100644 benchmarks/execution_throughput_bench.py create mode 100644 benchmarks/large_project_scaling_bench.py create mode 100644 docs/reference/scaling_baselines.md create mode 100644 features/scaling_performance.feature create mode 100644 features/steps/scaling_performance_steps.py create mode 100644 robot/helper_scaling_performance.py create mode 100644 robot/scaling_performance.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e75eeee..9f5f92b6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Added large-project scaling performance tests with ASV benchmarks for + context assembly, execution throughput, and project scaling. Includes + Behave BDD scenarios (78 scenarios, 200 steps), Robot Framework tests, + and baseline threshold fixtures. (#576) - Added TDD bug-capture tests for #969 — `plan correct` expects `decision_id` but M3 acceptance test passes `plan_id`. Behave BDD scenarios (revert and append modes) and Robot Framework integration tests verify that diff --git a/benchmarks/bench_uko_layer3.py b/benchmarks/bench_uko_layer3.py index b0298b0b0..d0d11cc40 100644 --- a/benchmarks/bench_uko_layer3.py +++ b/benchmarks/bench_uko_layer3.py @@ -20,11 +20,11 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) -from cleveragents.acms.uko.layer3_java import JAVA_DETAIL_LEVELS # noqa: E402 -from cleveragents.acms.uko.layer3_py import PYTHON_DETAIL_LEVELS # noqa: E402 -from cleveragents.acms.uko.layer3_rs import RUST_DETAIL_LEVELS # noqa: E402 -from cleveragents.acms.uko.layer3_ts import TYPESCRIPT_DETAIL_LEVELS # noqa: E402 -from cleveragents.acms.uko.vocabulary import ( # noqa: E402 +from cleveragents.acms.uko.layer3_java import JAVA_DETAIL_LEVELS +from cleveragents.acms.uko.layer3_py import PYTHON_DETAIL_LEVELS +from cleveragents.acms.uko.layer3_rs import RUST_DETAIL_LEVELS +from cleveragents.acms.uko.layer3_ts import TYPESCRIPT_DETAIL_LEVELS +from cleveragents.acms.uko.vocabulary import ( OO_EFFECTIVE_LEVELS, ProvenanceInfo, UKOClass, diff --git a/benchmarks/context_assembly_scaling_bench.py b/benchmarks/context_assembly_scaling_bench.py new file mode 100644 index 000000000..e5fea8559 --- /dev/null +++ b/benchmarks/context_assembly_scaling_bench.py @@ -0,0 +1,138 @@ +"""ASV benchmarks for ACMS context assembly at production scale. + +Extends the existing 1K-fragment ceiling from :mod:`acms_pipeline_bench` +to 5K and 10K fragments, exercising the full 10-stage pipeline, +deduplication + scoring, and knapsack budget packing. +""" + +from __future__ import annotations + +import importlib +import sys +import time +from pathlib import Path +from typing import ClassVar + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.application.services.acms_service import ACMSPipeline # noqa: E402 +from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 + ContextBudget, + ContextFragment, + FragmentProvenance, +) + +# Default provenance for benchmark fragments. +_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://scale") + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_fragments(count: int) -> list[ContextFragment]: + """Build *count* benchmark fragments with varied scores and tiers.""" + tiers = ("hot", "warm", "cold") + return [ + ContextFragment( + uko_node=f"bench://scale/{i}", + content=f"fragment content block {i} " * 5, + relevance_score=round((i % 10) / 10, 1), + token_count=50, + tier=tiers[i % 3], + provenance=_DEFAULT_PROV, + ) + for i in range(count) + ] + + +# --------------------------------------------------------------------------- +# Parameterized context assembly suite +# --------------------------------------------------------------------------- + + +class ContextAssemblyScalingSuite: + """Benchmark ACMS context assembly at production fragment counts.""" + + params: ClassVar[list[int]] = [100, 1_000, 5_000, 10_000] + param_names: ClassVar[list[str]] = ["fragment_count"] + timeout = 300 # 5 min for 10K fragments + + _pipeline: ACMSPipeline + _budget: ContextBudget + _fragments: list[ContextFragment] + + def setup(self, fragment_count: int) -> None: + """Create *fragment_count* ContextFragment objects.""" + self._pipeline = ACMSPipeline() + self._budget = ContextBudget(max_tokens=500_000, reserved_tokens=0) + self._fragments = _make_fragments(fragment_count) + + # -- timing methods ----------------------------------------------------- + + def time_full_pipeline(self, fragment_count: int) -> None: + """Time the full ACMS pipeline with *fragment_count* fragments.""" + self._pipeline.assemble( + plan_id="01JQBENCHM00000000000000AA", + fragments=self._fragments, + budget=self._budget, + ) + + def time_tiered_strategy(self, fragment_count: int) -> None: + """Time tiered strategy assembly.""" + self._pipeline.assemble( + plan_id="01JQBENCHM00000000000000AA", + fragments=self._fragments, + budget=self._budget, + strategy="tiered", + ) + + def time_recency_strategy(self, fragment_count: int) -> None: + """Time recency strategy assembly.""" + self._pipeline.assemble( + plan_id="01JQBENCHM00000000000000AA", + fragments=self._fragments, + budget=self._budget, + strategy="recency", + ) + + # -- tracking methods --------------------------------------------------- + + def track_assembled_tokens(self, fragment_count: int) -> int: + """Track total tokens in assembled context.""" + result = self._pipeline.assemble( + plan_id="01JQBENCHM00000000000000AA", + fragments=self._fragments, + budget=self._budget, + ) + return result.total_tokens + + def track_fragments_per_second(self, fragment_count: int) -> float: + """Track assembly throughput in fragments / second.""" + t0 = time.perf_counter() + self._pipeline.assemble( + plan_id="01JQBENCHM00000000000000AA", + fragments=self._fragments, + budget=self._budget, + ) + elapsed = time.perf_counter() - t0 + if elapsed <= 0: + return float(fragment_count) + return fragment_count / elapsed + + +# Attach ASV unit metadata without ``# type: ignore``. +setattr( # noqa: B010 + ContextAssemblyScalingSuite.track_assembled_tokens, "unit", "tokens" +) +setattr( # noqa: B010 + ContextAssemblyScalingSuite.track_fragments_per_second, "unit", "frags/s" +) diff --git a/benchmarks/execution_throughput_bench.py b/benchmarks/execution_throughput_bench.py new file mode 100644 index 000000000..b88540972 --- /dev/null +++ b/benchmarks/execution_throughput_bench.py @@ -0,0 +1,123 @@ +"""ASV benchmarks for plan execution throughput at scale. + +Measures sequential and concurrent plan execution overhead at varying +plan counts (10, 50, 100). Uses the lightweight in-process executor +path (no database, no LLM) to isolate execution-dispatch cost. +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from typing import ClassVar +from unittest.mock import MagicMock + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from ulid import ULID # noqa: E402 + +from cleveragents.application.services.plan_execution_context import ( # noqa: E402 + PlanExecutionContext, + RuntimeExecuteActor, +) +from cleveragents.application.services.plan_executor import ( # noqa: E402 + PlanExecutor, + StrategyDecision, +) +from cleveragents.domain.models.core.change import ( # noqa: E402 + InMemoryChangeSetStore, +) +from cleveragents.tool.registry import ToolRegistry # noqa: E402 +from cleveragents.tool.runner import ToolRunner # noqa: E402 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_RESOURCE_ID = "01HGZ6FE0AQDYTR4BXVQZ6EB00" + + +def _make_runner() -> ToolRunner: + return ToolRunner(registry=ToolRegistry()) + + +def _make_decisions(count: int) -> list[StrategyDecision]: + """Build a linear chain of *count* decisions.""" + root_id = str(ULID()) + return [ + StrategyDecision( + decision_id=root_id if i == 0 else str(ULID()), + step_text=f"Step {i + 1}", + sequence=i, + parent_id=root_id if i > 0 else None, + ) + for i in range(count) + ] + + +def _execute_single_plan(runner: ToolRunner) -> None: + """Execute one plan with 3 decisions (fire-and-forget).""" + plan_id = str(ULID()) + ctx = PlanExecutionContext( + plan_id=plan_id, + changeset_store=InMemoryChangeSetStore(), + ) + actor = RuntimeExecuteActor(tool_runner=runner, execution_context=ctx) + actor.execute(decisions=_make_decisions(3)) + + +# --------------------------------------------------------------------------- +# Parameterized execution throughput suite +# --------------------------------------------------------------------------- + + +class ExecutionThroughputSuite: + """Benchmark plan execution throughput at varying plan counts.""" + + params: ClassVar[list[int]] = [10, 50, 100] + param_names: ClassVar[list[str]] = ["plan_count"] + timeout = 300 + + _runner: ToolRunner + + def setup(self, plan_count: int) -> None: + """Prepare a shared tool runner.""" + self._runner = _make_runner() + + def time_sequential_plans(self, plan_count: int) -> None: + """Execute *plan_count* plans sequentially.""" + for _ in range(plan_count): + _execute_single_plan(self._runner) + + def time_executor_construction(self, plan_count: int) -> None: + """Construct *plan_count* PlanExecutor instances.""" + lifecycle = MagicMock() + for _ in range(plan_count): + ctx = PlanExecutionContext( + plan_id=str(ULID()), + changeset_store=InMemoryChangeSetStore(), + ) + PlanExecutor( + lifecycle_service=lifecycle, + tool_runner=self._runner, + execution_context=ctx, + ) + + def time_decision_tree_scaling(self, plan_count: int) -> None: + """Execute one plan with *plan_count* decisions.""" + plan_id = str(ULID()) + ctx = PlanExecutionContext( + plan_id=plan_id, + changeset_store=InMemoryChangeSetStore(), + ) + actor = RuntimeExecuteActor(tool_runner=self._runner, execution_context=ctx) + actor.execute(decisions=_make_decisions(plan_count)) diff --git a/benchmarks/large_project_scaling_bench.py b/benchmarks/large_project_scaling_bench.py new file mode 100644 index 000000000..67cbc595a --- /dev/null +++ b/benchmarks/large_project_scaling_bench.py @@ -0,0 +1,181 @@ +"""ASV benchmarks for large-project indexing at production scale. + +Measures the performance of ``walk_and_index`` and incremental refresh +at 1K, 10K, 50K, and 100K file counts, tracking throughput (files and +tokens per second) and indexed-file totals. + +Extends the existing 5K-file ceiling from +:mod:`large_project_decompose_bench` to production-scale repositories. +""" + +from __future__ import annotations + +import importlib +import os +import random +import shutil +import sys +import tempfile +import time +from pathlib import Path +from typing import ClassVar + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.application.services.repo_indexing_utils import ( # noqa: E402 + walk_and_index, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Realistic directory structure skeleton used for synthetic repos. +_DIRS = ("src", "tests", "docs", "scripts", "config") + +# Extension → content template. Keeps files realistic so that +# language detection and token estimation exercise real code paths. +_EXTENSIONS: dict[str, str] = { + ".py": "# auto-generated\ndef func_{i}() -> int:\n return {i}\n", + ".md": "# Document {i}\n\nGenerated documentation paragraph.\n", + ".json": '{{"id": {i}, "name": "item_{i}"}}\n', + ".ts": "export const value_{i}: number = {i};\n", + ".yaml": "key_{i}: value_{i}\n", +} + +_EXT_LIST = list(_EXTENSIONS.keys()) + + +def _build_project(root: str, file_count: int) -> None: + """Populate *root* with *file_count* files in a realistic layout.""" + rng = random.Random(42) # deterministic seed + for i in range(file_count): + subdir = _DIRS[i % len(_DIRS)] + # Create two levels of nesting to mimic real projects. + nested = f"pkg{i % 20}" + dirpath = os.path.join(root, subdir, nested) + os.makedirs(dirpath, exist_ok=True) + ext = _EXT_LIST[rng.randint(0, len(_EXT_LIST) - 1)] + template = _EXTENSIONS[ext] + content = template.format(i=i) + fpath = os.path.join(dirpath, f"file_{i:06d}{ext}") + with open(fpath, "w") as fh: + fh.write(content) + + +def _modify_subset(root: str, file_count: int, pct: float = 0.01) -> int: + """Modify *pct* of files under *root* and return count modified.""" + rng = random.Random(99) # deterministic seed + modify_count = max(1, int(file_count * pct)) + indices = rng.sample(range(file_count), modify_count) + modified = 0 + for i in indices: + subdir = _DIRS[i % len(_DIRS)] + nested = f"pkg{i % 20}" + ext = _EXT_LIST[rng.randint(0, len(_EXT_LIST) - 1)] + fpath = os.path.join(root, subdir, nested, f"file_{i:06d}{ext}") + if os.path.exists(fpath): + with open(fpath, "a") as fh: + fh.write(f"\n# modified at {time.monotonic()}\n") + modified += 1 + return modified + + +# --------------------------------------------------------------------------- +# Parameterized indexing suite +# --------------------------------------------------------------------------- + +_DEFAULT_INCLUDE: tuple[str, ...] = () +_DEFAULT_EXCLUDE: tuple[str, ...] = ("*.pyc", "__pycache__/*") + + +class IndexingScalingSuite: + """Benchmark ``walk_and_index`` at production-scale file counts.""" + + params: ClassVar[list[int]] = [1_000, 10_000, 50_000, 100_000] + param_names: ClassVar[list[str]] = ["file_count"] + timeout = 600 # 10 min for 100K files + + _tmpdir: str + + def setup(self, file_count: int) -> None: + """Create a temp directory with *file_count* files.""" + self._tmpdir = tempfile.mkdtemp(prefix="bench-scale-idx-") + _build_project(self._tmpdir, file_count) + + def teardown(self, file_count: int) -> None: + shutil.rmtree(self._tmpdir, ignore_errors=True) + + # -- timing methods ----------------------------------------------------- + + def time_walk_and_index(self, file_count: int) -> None: + """Time full ``walk_and_index`` for *file_count* files.""" + walk_and_index( + root=Path(self._tmpdir), + include_globs=_DEFAULT_INCLUDE, + exclude_globs=_DEFAULT_EXCLUDE, + max_file_size=None, + max_total_size=None, + ) + + def time_incremental_refresh(self, file_count: int) -> None: + """Modify 1 %% of files, then re-index the full tree. + + This measures incremental *walk* overhead; the actual diff merge + happens in ``RepoIndexingService.refresh_index`` which requires + a database. Here we just re-walk to quantify I/O cost. + """ + _modify_subset(self._tmpdir, file_count, pct=0.01) + walk_and_index( + root=Path(self._tmpdir), + include_globs=_DEFAULT_INCLUDE, + exclude_globs=_DEFAULT_EXCLUDE, + max_file_size=None, + max_total_size=None, + ) + + # -- tracking methods --------------------------------------------------- + + def track_indexed_file_count(self, file_count: int) -> int: + """Track the number of files actually indexed.""" + records = walk_and_index( + root=Path(self._tmpdir), + include_globs=_DEFAULT_INCLUDE, + exclude_globs=_DEFAULT_EXCLUDE, + max_file_size=None, + max_total_size=None, + ) + return len(records) + + def track_tokens_per_second(self, file_count: int) -> float: + """Track indexing throughput in tokens / second.""" + t0 = time.perf_counter() + records = walk_and_index( + root=Path(self._tmpdir), + include_globs=_DEFAULT_INCLUDE, + exclude_globs=_DEFAULT_EXCLUDE, + max_file_size=None, + max_total_size=None, + ) + elapsed = time.perf_counter() - t0 + total_tokens = sum(r.token_count for r in records) + if elapsed <= 0: + return float(total_tokens) + return total_tokens / elapsed + + +# Attach ASV unit metadata without ``# type: ignore``. +setattr( # noqa: B010 + IndexingScalingSuite.track_indexed_file_count, "unit", "files" +) +setattr( # noqa: B010 + IndexingScalingSuite.track_tokens_per_second, "unit", "tokens/s" +) diff --git a/docs/reference/scaling_baselines.md b/docs/reference/scaling_baselines.md new file mode 100644 index 000000000..73518fa9e --- /dev/null +++ b/docs/reference/scaling_baselines.md @@ -0,0 +1,105 @@ +# Scaling Baselines + +Performance baselines and expectations for CleverAgents at production scale. + +## Overview + +CleverAgents is designed to handle repositories ranging from small +microservices (1K files) to large enterprise monorepos (100K+ files). +This document captures the baseline performance thresholds and scaling +expectations used by the CI benchmark suite. + +## Scale Profiles + +| Profile | Files | Size (MB) | Index Time (s) | Decompose Time (s) | +|----------|---------|-----------|-----------------|---------------------| +| small | 1,000 | 50 | 5 | 10 | +| medium | 5,000 | 250 | 25 | 50 | +| large | 10,000 | 500 | 50 | 100 | +| xlarge | 50,000 | 2,500 | 200 | 400 | +| xxlarge | 100,000 | 5,000 | 450 | 900 | + +## Indexing Thresholds + +Latency percentiles for `walk_and_index` on local SSD with 8+ GB RAM: + +| File Count | p50 (ms) | p95 (ms) | p99 (ms) | Max (ms) | +|------------|----------|----------|-----------|-----------| +| 1,000 | 2,000 | 5,000 | 8,000 | 12,000 | +| 5,000 | 10,000 | 25,000 | 40,000 | 60,000 | +| 10,000 | 18,000 | 45,000 | 72,000 | 110,000 | +| 50,000 | 75,000 | 180,000 | 290,000 | 450,000 | +| 100,000 | 140,000 | 340,000 | 550,000 | 850,000 | + +## Decomposition Thresholds + +Decomposition typically takes 2x indexing time: + +| File Count | p50 (ms) | p95 (ms) | p99 (ms) | Max (ms) | +|------------|----------|-----------|------------|------------| +| 1,000 | 4,000 | 10,000 | 16,000 | 24,000 | +| 5,000 | 20,000 | 50,000 | 80,000 | 120,000 | +| 10,000 | 40,000 | 100,000 | 160,000 | 240,000 | +| 50,000 | 160,000 | 400,000 | 640,000 | 960,000 | +| 100,000 | 300,000 | 750,000 | 1,200,000 | 1,800,000 | + +## Context Assembly Thresholds + +ACMS pipeline latency for varying fragment counts: + +| Fragments | p50 (ms) | p95 (ms) | p99 (ms) | +|-----------|----------|----------|----------| +| 100 | 50 | 150 | 300 | +| 1,000 | 500 | 1,500 | 3,000 | +| 5,000 | 2,500 | 7,500 | 15,000 | +| 10,000 | 5,000 | 15,000 | 30,000 | + +## Execution Throughput Thresholds + +Plan execution dispatch latency: + +| Plans | p50 (ms) | p95 (ms) | p99 (ms) | +|-------|----------|----------|----------| +| 10 | 200 | 500 | 1,000 | +| 50 | 1,000 | 2,500 | 5,000 | +| 100 | 2,000 | 5,000 | 10,000 | + +## Memory Usage + +| File Count | Peak (MB) | Steady State (MB) | +|------------|-----------|-------------------| +| 1,000 | 256 | 128 | +| 5,000 | 768 | 384 | +| 10,000 | 1,536 | 768 | +| 50,000 | 6,144 | 3,072 | +| 100,000 | 12,288 | 6,144 | + +## Scaling Expectations + +- **Indexing**: O(n log n) — sub-linear scaling expected +- **Decomposition**: O(n log n) — sub-linear scaling expected +- **Context Assembly**: O(n log n) — sub-linear for large fragment sets +- **Execution**: O(n) — linear scaling expected +- **Memory**: O(n) — linear scaling expected + +### Sub-linear Scaling Invariants + +The benchmark suite enforces these invariants: + +- 10K indexing p50 < 10x the 1K p50 +- 50K indexing p50 < 5x the 10K p50 +- 100K indexing p50 < 2.5x the 50K p50 + +## ASV Benchmark Suites + +| Suite | Params | Timeout | +|----------------------------------|-------------------------|---------| +| `IndexingScalingSuite` | 1K, 10K, 50K, 100K | 600s | +| `ContextAssemblyScalingSuite` | 100, 1K, 5K, 10K | 300s | +| `ExecutionThroughputSuite` | 10, 50, 100 | 300s | + +## Fixture Files + +- `features/fixtures/scale/scale_metadata.json` — profile definitions +- `features/fixtures/scale/baseline_thresholds.json` — p50/p95/p99 thresholds +- `features/fixtures/scale/generator_instructions.md` — fixture generation guide diff --git a/features/consolidated_misc.feature b/features/consolidated_misc.feature index 23da76bed..e52c36346 100644 --- a/features/consolidated_misc.feature +++ b/features/consolidated_misc.feature @@ -958,8 +958,8 @@ Feature: Consolidated Misc Scenario: Scale metadata contains all required profiles Given a scale test environment is ready When I scale test load the scale metadata fixture - Then the scale test metadata should have exactly 3 profiles - And the scale test profile names should be "small, medium, large" + Then the scale test metadata should have exactly 5 profiles + And the scale test profile names should be "small, medium, large, xlarge, xxlarge" Scenario: Each scale profile has required fields @@ -998,6 +998,8 @@ Feature: Consolidated Misc Then the scale test indexing thresholds should cover "1000_files" And the scale test indexing thresholds should cover "5000_files" And the scale test indexing thresholds should cover "10000_files" + And the scale test indexing thresholds should cover "50000_files" + And the scale test indexing thresholds should cover "100000_files" Scenario: Each indexing threshold has percentile fields @@ -1012,6 +1014,8 @@ Feature: Consolidated Misc Then the scale test decomposition thresholds should cover "1000_files" And the scale test decomposition thresholds should cover "5000_files" And the scale test decomposition thresholds should cover "10000_files" + And the scale test decomposition thresholds should cover "50000_files" + And the scale test decomposition thresholds should cover "100000_files" Scenario: Each decomposition threshold has percentile fields diff --git a/features/fixtures/scale/baseline_thresholds.json b/features/fixtures/scale/baseline_thresholds.json index 9fac39c5a..71598dafe 100644 --- a/features/fixtures/scale/baseline_thresholds.json +++ b/features/fixtures/scale/baseline_thresholds.json @@ -22,6 +22,20 @@ "p99_ms": 72000, "max_ms": 110000, "notes": "Large repo: enterprise monorepo scale; sub-linear scaling expected" + }, + "50000_files": { + "p50_ms": 75000, + "p95_ms": 180000, + "p99_ms": 290000, + "max_ms": 450000, + "notes": "XLarge repo: multi-team enterprise monorepo; sub-linear scaling expected" + }, + "100000_files": { + "p50_ms": 140000, + "p95_ms": 340000, + "p99_ms": 550000, + "max_ms": 850000, + "notes": "XXLarge repo: maximum production scale; sub-linear scaling expected" } }, "decomposition": { @@ -45,6 +59,20 @@ "p99_ms": 160000, "max_ms": 240000, "notes": "Decomposition typically 2x indexing time" + }, + "50000_files": { + "p50_ms": 160000, + "p95_ms": 400000, + "p99_ms": 640000, + "max_ms": 960000, + "notes": "Decomposition typically 2x indexing time; sub-linear scaling" + }, + "100000_files": { + "p50_ms": 300000, + "p95_ms": 750000, + "p99_ms": 1200000, + "max_ms": 1800000, + "notes": "Decomposition typically 2x indexing time; sub-linear scaling" } }, "memory_usage_mb": { @@ -59,11 +87,67 @@ "10000_files": { "peak_mb": 1536, "steady_state_mb": 768 + }, + "50000_files": { + "peak_mb": 6144, + "steady_state_mb": 3072 + }, + "100000_files": { + "peak_mb": 12288, + "steady_state_mb": 6144 + } + }, + "context_assembly": { + "100_fragments": { + "p50_ms": 50, + "p95_ms": 150, + "p99_ms": 300, + "notes": "Baseline context assembly at small scale" + }, + "1000_fragments": { + "p50_ms": 500, + "p95_ms": 1500, + "p99_ms": 3000, + "notes": "Medium context assembly scale" + }, + "5000_fragments": { + "p50_ms": 2500, + "p95_ms": 7500, + "p99_ms": 15000, + "notes": "Large context assembly; linear scaling expected" + }, + "10000_fragments": { + "p50_ms": 5000, + "p95_ms": 15000, + "p99_ms": 30000, + "notes": "Maximum context assembly scale" + } + }, + "execution_throughput": { + "10_plans": { + "p50_ms": 200, + "p95_ms": 500, + "p99_ms": 1000, + "notes": "Light execution workload" + }, + "50_plans": { + "p50_ms": 1000, + "p95_ms": 2500, + "p99_ms": 5000, + "notes": "Medium execution workload" + }, + "100_plans": { + "p50_ms": 2000, + "p95_ms": 5000, + "p99_ms": 10000, + "notes": "Heavy execution workload" } }, "scaling_expectations": { "indexing_complexity": "O(n log n)", "decomposition_complexity": "O(n log n)", + "context_assembly_complexity": "O(n log n)", + "execution_complexity": "O(n)", "memory_complexity": "O(n)", "notes": "Thresholds assume local SSD storage and 8+ GB RAM available" } diff --git a/features/fixtures/scale/scale_metadata.json b/features/fixtures/scale/scale_metadata.json index 6f84c98d7..32edbc9b8 100644 --- a/features/fixtures/scale/scale_metadata.json +++ b/features/fixtures/scale/scale_metadata.json @@ -43,6 +43,39 @@ "expected_index_time_s": 50.0, "expected_decomposition_time_s": 100.0, "description": "Large repository representative of a full enterprise monorepo" + }, + { + "name": "xlarge", + "file_count": 50000, + "total_size_mb": 2500, + "language_mix": { + "python": 0.35, + "javascript": 0.2, + "typescript": 0.2, + "yaml": 0.1, + "json": 0.1, + "markdown": 0.05 + }, + "expected_index_time_s": 200.0, + "expected_decomposition_time_s": 400.0, + "description": "Extra-large repository representative of a multi-team enterprise monorepo" + }, + { + "name": "xxlarge", + "file_count": 100000, + "total_size_mb": 5000, + "language_mix": { + "python": 0.3, + "javascript": 0.2, + "typescript": 0.2, + "yaml": 0.1, + "json": 0.1, + "markdown": 0.05, + "html": 0.05 + }, + "expected_index_time_s": 450.0, + "expected_decomposition_time_s": 900.0, + "description": "Maximum-scale repository representative of a large enterprise codebase" } ], "supported_languages": [ diff --git a/features/scaling_performance.feature b/features/scaling_performance.feature new file mode 100644 index 000000000..14bf05781 --- /dev/null +++ b/features/scaling_performance.feature @@ -0,0 +1,100 @@ +Feature: Large project scaling performance baselines + Validates that benchmark fixtures, threshold definitions, and scaling + expectations are present and self-consistent for 50K and 100K file + scales, and that context assembly and execution throughput thresholds + are defined. + + Background: + Given a scaling performance test environment is ready + + # ────────────────────────────────────────────────── + # Section 1: Extended scale profile validation + # ────────────────────────────────────────────────── + + Scenario: Scale metadata includes xlarge and xxlarge profiles + When I scaling perf load the scale metadata fixture + Then the scaling perf metadata should contain profile "xlarge" + And the scaling perf metadata should contain profile "xxlarge" + + Scenario: XLarge profile has correct file count + When I scaling perf load the scale metadata fixture + Then the scaling perf profile "xlarge" should have file count 50000 + + Scenario: XXLarge profile has correct file count + When I scaling perf load the scale metadata fixture + Then the scaling perf profile "xxlarge" should have file count 100000 + + Scenario: New profiles have valid language mixes + When I scaling perf load the scale metadata fixture + Then the scaling perf language mix should sum to 1.0 for profile "xlarge" + And the scaling perf language mix should sum to 1.0 for profile "xxlarge" + + # ────────────────────────────────────────────────── + # Section 2: Extended threshold validation + # ────────────────────────────────────────────────── + + Scenario: Indexing thresholds cover 50K and 100K files + When I scaling perf load the baseline thresholds fixture + Then the scaling perf indexing thresholds should cover "50000_files" + And the scaling perf indexing thresholds should cover "100000_files" + + Scenario: Decomposition thresholds cover 50K and 100K files + When I scaling perf load the baseline thresholds fixture + Then the scaling perf decomposition thresholds should cover "50000_files" + And the scaling perf decomposition thresholds should cover "100000_files" + + Scenario: Extended thresholds are monotonically increasing + When I scaling perf load the baseline thresholds fixture + Then scaling perf p50 should be less than p95 for "50000_files" indexing + And scaling perf p95 should be less than p99 for "50000_files" indexing + And scaling perf p50 should be less than p95 for "100000_files" indexing + And scaling perf p95 should be less than p99 for "100000_files" indexing + + Scenario: 50K indexing threshold scales sub-linearly from 10K + When I scaling perf load the baseline thresholds fixture + Then scaling perf 50K indexing p50 should be less than 5x the 10K p50 + + Scenario: 100K indexing threshold scales sub-linearly from 50K + When I scaling perf load the baseline thresholds fixture + Then scaling perf 100K indexing p50 should be less than 2.5x the 50K p50 + + # ────────────────────────────────────────────────── + # Section 3: Context assembly thresholds + # ────────────────────────────────────────────────── + + Scenario: Context assembly thresholds section exists + When I scaling perf load the baseline thresholds fixture + Then the scaling perf thresholds should have context assembly section + + Scenario: Context assembly covers 5K and 10K fragments + When I scaling perf load the baseline thresholds fixture + Then the scaling perf context assembly thresholds should cover "5000_fragments" + And the scaling perf context assembly thresholds should cover "10000_fragments" + + # ────────────────────────────────────────────────── + # Section 4: Execution throughput thresholds + # ────────────────────────────────────────────────── + + Scenario: Execution throughput thresholds section exists + When I scaling perf load the baseline thresholds fixture + Then the scaling perf thresholds should have execution throughput section + + Scenario: Execution throughput covers all plan counts + When I scaling perf load the baseline thresholds fixture + Then the scaling perf execution thresholds should cover "10_plans" + And the scaling perf execution thresholds should cover "50_plans" + And the scaling perf execution thresholds should cover "100_plans" + + # ────────────────────────────────────────────────── + # Section 5: Memory thresholds at production scale + # ────────────────────────────────────────────────── + + Scenario: Memory thresholds cover 50K and 100K files + When I scaling perf load the baseline thresholds fixture + Then the scaling perf memory thresholds should cover "50000_files" + And the scaling perf memory thresholds should cover "100000_files" + + Scenario: Memory peak exceeds steady state for extended profiles + When I scaling perf load the baseline thresholds fixture + Then scaling perf memory peak should exceed steady state for "50000_files" + And scaling perf memory peak should exceed steady state for "100000_files" diff --git a/features/steps/scaling_performance_steps.py b/features/steps/scaling_performance_steps.py new file mode 100644 index 000000000..c683f8040 --- /dev/null +++ b/features/steps/scaling_performance_steps.py @@ -0,0 +1,216 @@ +"""Step definitions for large-project scaling performance validation.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +from behave import given, then, when +from behave.runner import Context + +_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "scale" + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a scaling performance test environment is ready") +def step_scaling_perf_env_ready(context: Context) -> None: + """Verify scale test fixtures directory exists.""" + assert _FIXTURES_DIR.is_dir(), f"Fixtures dir missing: {_FIXTURES_DIR}" + context.sp_metadata = None + context.sp_thresholds = None + + +# --------------------------------------------------------------------------- +# Fixture loading +# --------------------------------------------------------------------------- + + +@when("I scaling perf load the scale metadata fixture") +def step_scaling_perf_load_metadata(context: Context) -> None: + path = _FIXTURES_DIR / "scale_metadata.json" + assert path.exists(), f"Missing fixture: {path}" + with open(path) as fh: + context.sp_metadata = json.load(fh) + + +@when("I scaling perf load the baseline thresholds fixture") +def step_scaling_perf_load_thresholds(context: Context) -> None: + path = _FIXTURES_DIR / "baseline_thresholds.json" + assert path.exists(), f"Missing fixture: {path}" + with open(path) as fh: + context.sp_thresholds = json.load(fh) + + +# --------------------------------------------------------------------------- +# Profile validation +# --------------------------------------------------------------------------- + + +def _get_profile(context: Context, name: str) -> dict[str, Any]: + """Return the profile dict with the given name, or fail.""" + assert context.sp_metadata is not None, "Metadata not loaded" + for p in context.sp_metadata["scale_profiles"]: + if p["name"] == name: + return p + raise AssertionError(f"Profile {name!r} not found in metadata") + + +@then('the scaling perf metadata should contain profile "{name}"') +def step_scaling_perf_has_profile(context: Context, name: str) -> None: + _get_profile(context, name) + + +@then('the scaling perf profile "{name}" should have file count {count:d}') +def step_scaling_perf_profile_file_count( + context: Context, name: str, count: int +) -> None: + profile = _get_profile(context, name) + actual = profile["file_count"] + assert actual == count, f"Profile {name}: expected {count}, got {actual}" + + +@then('the scaling perf language mix should sum to 1.0 for profile "{name}"') +def step_scaling_perf_lang_mix_sums(context: Context, name: str) -> None: + profile = _get_profile(context, name) + total = sum(profile["language_mix"].values()) + assert math.isclose(total, 1.0, rel_tol=1e-6), ( + f"Language mix for {name} sums to {total}, expected 1.0" + ) + + +# --------------------------------------------------------------------------- +# Threshold coverage +# --------------------------------------------------------------------------- + + +@then('the scaling perf indexing thresholds should cover "{file_count}"') +def step_scaling_perf_indexing_covers(context: Context, file_count: str) -> None: + assert context.sp_thresholds is not None + assert file_count in context.sp_thresholds["indexing"], ( + f"Missing indexing threshold for {file_count}" + ) + + +@then('the scaling perf decomposition thresholds should cover "{file_count}"') +def step_scaling_perf_decomp_covers(context: Context, file_count: str) -> None: + assert context.sp_thresholds is not None + assert file_count in context.sp_thresholds["decomposition"], ( + f"Missing decomposition threshold for {file_count}" + ) + + +# --------------------------------------------------------------------------- +# Monotonicity +# --------------------------------------------------------------------------- + + +@then('scaling perf p50 should be less than p95 for "{key}" indexing') +def step_scaling_perf_p50_lt_p95(context: Context, key: str) -> None: + assert context.sp_thresholds is not None + t = context.sp_thresholds["indexing"][key] + assert t["p50_ms"] < t["p95_ms"], ( + f"p50 ({t['p50_ms']}) >= p95 ({t['p95_ms']}) for {key}" + ) + + +@then('scaling perf p95 should be less than p99 for "{key}" indexing') +def step_scaling_perf_p95_lt_p99(context: Context, key: str) -> None: + assert context.sp_thresholds is not None + t = context.sp_thresholds["indexing"][key] + assert t["p95_ms"] < t["p99_ms"], ( + f"p95 ({t['p95_ms']}) >= p99 ({t['p99_ms']}) for {key}" + ) + + +# --------------------------------------------------------------------------- +# Sub-linear scaling checks +# --------------------------------------------------------------------------- + + +@then("scaling perf 50K indexing p50 should be less than 5x the 10K p50") +def step_scaling_perf_50k_sublinear(context: Context) -> None: + assert context.sp_thresholds is not None + idx = context.sp_thresholds["indexing"] + p50_10k = idx["10000_files"]["p50_ms"] + p50_50k = idx["50000_files"]["p50_ms"] + assert p50_50k < 5 * p50_10k, ( + f"50K p50 ({p50_50k}) should be < 5x 10K p50 ({5 * p50_10k})" + ) + + +@then("scaling perf 100K indexing p50 should be less than 2.5x the 50K p50") +def step_scaling_perf_100k_sublinear(context: Context) -> None: + assert context.sp_thresholds is not None + idx = context.sp_thresholds["indexing"] + p50_50k = idx["50000_files"]["p50_ms"] + p50_100k = idx["100000_files"]["p50_ms"] + limit = int(2.5 * p50_50k) + assert p50_100k < limit, f"100K p50 ({p50_100k}) should be < 2.5x 50K p50 ({limit})" + + +# --------------------------------------------------------------------------- +# Context assembly +# --------------------------------------------------------------------------- + + +@then("the scaling perf thresholds should have context assembly section") +def step_scaling_perf_has_ctx_assembly(context: Context) -> None: + assert context.sp_thresholds is not None + assert "context_assembly" in context.sp_thresholds + + +@then('the scaling perf context assembly thresholds should cover "{fragment_count}"') +def step_scaling_perf_ctx_assembly_covers( + context: Context, fragment_count: str +) -> None: + assert context.sp_thresholds is not None + assert fragment_count in context.sp_thresholds["context_assembly"], ( + f"Missing context_assembly threshold for {fragment_count}" + ) + + +# --------------------------------------------------------------------------- +# Execution throughput +# --------------------------------------------------------------------------- + + +@then("the scaling perf thresholds should have execution throughput section") +def step_scaling_perf_has_exec_throughput(context: Context) -> None: + assert context.sp_thresholds is not None + assert "execution_throughput" in context.sp_thresholds + + +@then('the scaling perf execution thresholds should cover "{plan_count}"') +def step_scaling_perf_exec_covers(context: Context, plan_count: str) -> None: + assert context.sp_thresholds is not None + assert plan_count in context.sp_thresholds["execution_throughput"], ( + f"Missing execution_throughput threshold for {plan_count}" + ) + + +# --------------------------------------------------------------------------- +# Memory thresholds +# --------------------------------------------------------------------------- + + +@then('the scaling perf memory thresholds should cover "{file_count}"') +def step_scaling_perf_memory_covers(context: Context, file_count: str) -> None: + assert context.sp_thresholds is not None + assert file_count in context.sp_thresholds["memory_usage_mb"], ( + f"Missing memory_usage_mb threshold for {file_count}" + ) + + +@then('scaling perf memory peak should exceed steady state for "{file_count}"') +def step_scaling_perf_memory_peak_gt_steady(context: Context, file_count: str) -> None: + assert context.sp_thresholds is not None + mem = context.sp_thresholds["memory_usage_mb"][file_count] + assert mem["peak_mb"] > mem["steady_state_mb"], ( + f"Peak ({mem['peak_mb']}) <= steady ({mem['steady_state_mb']}) for {file_count}" + ) diff --git a/robot/helper_scale_test.py b/robot/helper_scale_test.py index b8627f7e6..730262ab6 100644 --- a/robot/helper_scale_test.py +++ b/robot/helper_scale_test.py @@ -59,7 +59,7 @@ def _cmd_validate_profiles() -> int: "expected_index_time_s", "expected_decomposition_time_s", ] - expected_names = {"small", "medium", "large"} + expected_names = {"small", "medium", "large", "xlarge", "xxlarge"} actual_names = {p["name"] for p in data["scale_profiles"]} assert actual_names == expected_names, ( f"Expected profiles {expected_names}, got {actual_names}" @@ -82,7 +82,13 @@ def _cmd_validate_thresholds() -> int: assert "schema_version" in data, "Missing schema_version" for section in ("indexing", "decomposition"): assert section in data, f"Missing section: {section}" - for file_count in ("1000_files", "5000_files", "10000_files"): + for file_count in ( + "1000_files", + "5000_files", + "10000_files", + "50000_files", + "100000_files", + ): assert file_count in data[section], f"Missing {file_count} in {section}" for pct in ("p50_ms", "p95_ms", "p99_ms"): assert pct in data[section][file_count], ( diff --git a/robot/helper_scaling_performance.py b/robot/helper_scaling_performance.py new file mode 100644 index 000000000..112bb08d7 --- /dev/null +++ b/robot/helper_scaling_performance.py @@ -0,0 +1,207 @@ +"""Robot Framework helper for large-project scaling performance tests. + +Provides a CLI-style interface for Robot to invoke extended fixture +validation, threshold sub-linearity checks, and a small-scale indexing +throughput test. + +Usage: + python robot/helper_scaling_performance.py check-extended-profiles + python robot/helper_scaling_performance.py check-extended-thresholds + python robot/helper_scaling_performance.py check-context-assembly + python robot/helper_scaling_performance.py check-execution-throughput + python robot/helper_scaling_performance.py check-sublinear + python robot/helper_scaling_performance.py run-small-index +""" + +from __future__ import annotations + +import json +import math +import os +import shutil +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +# Ensure the src directory is on the import path. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "scale" + + +def _load_json(filename: str) -> dict[str, Any]: + """Load a JSON fixture file.""" + path = _FIXTURES_DIR / filename + if not path.exists(): + print(f"ERROR: fixture not found: {path}") + sys.exit(1) + with open(path) as fh: + result: dict[str, Any] = json.load(fh) + return result + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + + +def _cmd_check_extended_profiles() -> int: + """Verify xlarge and xxlarge profiles are present with valid mixes.""" + data = _load_json("scale_metadata.json") + names = {p["name"] for p in data["scale_profiles"]} + assert "xlarge" in names, "Missing xlarge profile" + assert "xxlarge" in names, "Missing xxlarge profile" + for profile in data["scale_profiles"]: + if profile["name"] in ("xlarge", "xxlarge"): + total = sum(profile["language_mix"].values()) + assert math.isclose(total, 1.0, rel_tol=1e-6), ( + f"Language mix for {profile['name']} sums to {total}" + ) + assert profile["file_count"] >= 50000, ( + f"File count for {profile['name']} too low: {profile['file_count']}" + ) + print("scaling-profiles-ok: xlarge and xxlarge profiles validated") + return 0 + + +def _cmd_check_extended_thresholds() -> int: + """Verify 50K and 100K indexing and decomposition thresholds.""" + data = _load_json("baseline_thresholds.json") + for section in ("indexing", "decomposition"): + for key in ("50000_files", "100000_files"): + assert key in data[section], f"Missing {key} in {section}" + t = data[section][key] + for pct in ("p50_ms", "p95_ms", "p99_ms"): + assert pct in t, f"Missing {pct} in {section}[{key}]" + assert t["p50_ms"] < t["p95_ms"] < t["p99_ms"], ( + f"Non-monotonic in {section}[{key}]" + ) + print("scaling-thresholds-ok: 50K and 100K thresholds validated") + return 0 + + +def _cmd_check_context_assembly() -> int: + """Verify context assembly threshold section exists.""" + data = _load_json("baseline_thresholds.json") + assert "context_assembly" in data, "Missing context_assembly section" + for key in ("5000_fragments", "10000_fragments"): + assert key in data["context_assembly"], f"Missing {key} in context_assembly" + print("scaling-ctx-assembly-ok: context assembly thresholds validated") + return 0 + + +def _cmd_check_execution_throughput() -> int: + """Verify execution throughput threshold section exists.""" + data = _load_json("baseline_thresholds.json") + assert "execution_throughput" in data, "Missing execution_throughput section" + for key in ("10_plans", "50_plans", "100_plans"): + assert key in data["execution_throughput"], ( + f"Missing {key} in execution_throughput" + ) + print("scaling-exec-throughput-ok: execution throughput thresholds validated") + return 0 + + +def _cmd_check_sublinear() -> int: + """Verify sub-linear scaling of extended thresholds.""" + data = _load_json("baseline_thresholds.json") + idx = data["indexing"] + p50_10k = idx["10000_files"]["p50_ms"] + p50_50k = idx["50000_files"]["p50_ms"] + p50_100k = idx["100000_files"]["p50_ms"] + # 50K should be less than 5x the 10K value (sub-linear) + assert p50_50k < 5 * p50_10k, f"50K p50 ({p50_50k}) >= 5x 10K p50 ({5 * p50_10k})" + # 100K should be less than 2.5x the 50K value (sub-linear) + limit = int(2.5 * p50_50k) + assert p50_100k < limit, f"100K p50 ({p50_100k}) >= 2.5x 50K p50 ({limit})" + print("scaling-sublinear-ok: sub-linear scaling verified") + return 0 + + +def _cmd_run_small_index() -> int: + """Create a 1K-file fixture, index it, verify throughput.""" + from cleveragents.application.services.repo_indexing_utils import walk_and_index + + tmpdir = tempfile.mkdtemp(prefix="robot-scale-idx-") + try: + # Create 1000 files + for i in range(1000): + subdir = os.path.join(tmpdir, f"pkg{i % 10}") + os.makedirs(subdir, exist_ok=True) + fpath = os.path.join(subdir, f"file_{i:04d}.py") + with open(fpath, "w") as fh: + fh.write(f"# module {i}\ndef func_{i}() -> int:\n return {i}\n") + + t0 = time.perf_counter() + records = walk_and_index( + root=Path(tmpdir), + include_globs=(), + exclude_globs=("*.pyc",), + max_file_size=None, + max_total_size=None, + ) + elapsed = time.perf_counter() - t0 + file_count = len(records) + total_tokens = sum(r.token_count for r in records) + throughput = total_tokens / elapsed if elapsed > 0 else 0.0 + + print(f" files_indexed: {file_count}") + print(f" total_tokens: {total_tokens}") + print(f" elapsed_s: {elapsed:.3f}") + print(f" tokens_per_second: {throughput:.0f}") + + # Minimum threshold: at least 500 files should be indexed + assert file_count >= 500, f"Only {file_count} files indexed, expected >= 500" + # Index should complete within the 1K threshold (12 seconds max) + assert elapsed < 12.0, f"Indexing took {elapsed:.1f}s, exceeds 12s threshold" + print("scaling-index-ok: small-scale indexing passed") + return 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Main dispatch +# --------------------------------------------------------------------------- + + +def main() -> int: + """Entry point called by Robot Framework ``Run Process``.""" + if len(sys.argv) < 2: + print( + "Usage: helper_scaling_performance.py " + "" + ) + return 1 + + command = sys.argv[1] + commands: dict[str, Any] = { + "check-extended-profiles": _cmd_check_extended_profiles, + "check-extended-thresholds": _cmd_check_extended_thresholds, + "check-context-assembly": _cmd_check_context_assembly, + "check-execution-throughput": _cmd_check_execution_throughput, + "check-sublinear": _cmd_check_sublinear, + "run-small-index": _cmd_run_small_index, + } + + handler = commands.get(command) + if handler is None: + print(f"Unknown command: {command}") + return 1 + + try: + result: int = handler() + return result + except (AssertionError, KeyError, json.JSONDecodeError) as exc: + print(f"FAIL: {exc}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/robot/scaling_performance.robot b/robot/scaling_performance.robot new file mode 100644 index 000000000..6cb8a7d46 --- /dev/null +++ b/robot/scaling_performance.robot @@ -0,0 +1,60 @@ +*** Settings *** +Documentation Large-project scaling performance integration tests. +... Validates that benchmark fixtures exist, thresholds are +... reasonable, and a small-scale indexing run meets minimum +... throughput targets. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_scaling_performance.py + +*** Test Cases *** +Verify Extended Scale Profiles Exist + [Documentation] Verify xlarge and xxlarge profiles are present in metadata + ${result}= Run Process ${PYTHON} ${HELPER} check-extended-profiles cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-profiles-ok + +Verify Extended Indexing Thresholds Exist + [Documentation] Verify 50K and 100K indexing thresholds are present + ${result}= Run Process ${PYTHON} ${HELPER} check-extended-thresholds cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-thresholds-ok + +Verify Context Assembly Thresholds Exist + [Documentation] Verify context assembly threshold section and entries + ${result}= Run Process ${PYTHON} ${HELPER} check-context-assembly cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-ctx-assembly-ok + +Verify Execution Throughput Thresholds Exist + [Documentation] Verify execution throughput threshold section and entries + ${result}= Run Process ${PYTHON} ${HELPER} check-execution-throughput cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-exec-throughput-ok + +Verify Sub-Linear Scaling Of Extended Thresholds + [Documentation] Verify 50K and 100K thresholds scale sub-linearly + ${result}= Run Process ${PYTHON} ${HELPER} check-sublinear cwd=${WORKSPACE} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-sublinear-ok + +Small Scale Indexing Throughput Check + [Documentation] Create a 1K-file fixture, index it, verify throughput + ${result}= Run Process ${PYTHON} ${HELPER} run-small-index cwd=${WORKSPACE} timeout=180s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scaling-index-ok -- 2.52.0 From 344f05ebb0302fe9fb8c60e1a872a708c1d373d8 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Sat, 21 Mar 2026 04:30:49 +0000 Subject: [PATCH 2/2] ci: trigger CI re-run -- 2.52.0