forked from cleveragents/cleveragents-core
brent.edwards
b88bc0ec1b
## Summary Add large project scaling benchmarks and tests at production scale (10K–100K files). ### New ASV Benchmarks **IndexingScalingSuite** (`large_project_scaling_bench.py`): - `time_walk_and_index` at 1K/10K/50K/100K files - `time_incremental_refresh` (1% modified files) - `track_indexed_file_count`, `track_tokens_per_second` **ContextAssemblyScalingSuite** (`context_assembly_scaling_bench.py`): - `time_full_pipeline` at 100/1K/5K/10K fragments - `time_tiered_strategy`, `time_recency_strategy` - `track_assembled_tokens`, `track_fragments_per_second` **ExecutionThroughputSuite** (`execution_throughput_bench.py`): - `time_sequential_plans` at 10/50/100 plans - `time_executor_construction`, `time_decision_tree_scaling` ### Scale Fixture Updates - Added `xlarge` (50K files) and `xxlarge` (100K files) profiles to `scale_metadata.json` - Added 50K/100K thresholds to `baseline_thresholds.json` - Added `context_assembly` and `execution_throughput` threshold sections ### Tests & Documentation - 15 Behave scenarios validating profiles, thresholds, monotonicity, memory budgets - 6 Robot integration tests including live 1K-file indexing throughput check - `docs/reference/scaling_baselines.md` documenting all baseline metrics ### Quality Gates | Session | Result | |---|---| | `nox -s lint` | PASS | | `nox -s typecheck` | PASS (0 errors) | | `nox -s unit_tests` | PASS (10,910 scenarios) | | `nox -s integration_tests` | PASS (1,526 tests) | | `nox -s coverage_report` | 97% (>= 97%) | Closes #859 Reviewed-on: cleveragents/cleveragents-core#984 Co-authored-by: Brent E. Edwards <brent.edwards@cleverthis.com> Co-committed-by: Brent E. Edwards <brent.edwards@cleverthis.com>
155 lines
3.8 KiB
JSON
155 lines
3.8 KiB
JSON
{
|
|
"schema_version": "1.0",
|
|
"description": "Baseline performance thresholds for scale testing. Values are in milliseconds unless noted otherwise.",
|
|
"indexing": {
|
|
"1000_files": {
|
|
"p50_ms": 2000,
|
|
"p95_ms": 5000,
|
|
"p99_ms": 8000,
|
|
"max_ms": 12000,
|
|
"notes": "Small repo: single microservice scale"
|
|
},
|
|
"5000_files": {
|
|
"p50_ms": 10000,
|
|
"p95_ms": 25000,
|
|
"p99_ms": 40000,
|
|
"max_ms": 60000,
|
|
"notes": "Medium repo: multi-service monorepo scale"
|
|
},
|
|
"10000_files": {
|
|
"p50_ms": 18000,
|
|
"p95_ms": 45000,
|
|
"p99_ms": 72000,
|
|
"max_ms": 110000,
|
|
"notes": "Large repo: enterprise monorepo scale; sub-linear scaling expected"
|
|
},
|
|
"50000_files": {
|
|
"p50_ms": 75000,
|
|
"p95_ms": 180000,
|
|
"p99_ms": 290000,
|
|
"max_ms": 450000,
|
|
"notes": "XLarge repo: multi-team enterprise monorepo; sub-linear scaling expected"
|
|
},
|
|
"100000_files": {
|
|
"p50_ms": 140000,
|
|
"p95_ms": 340000,
|
|
"p99_ms": 550000,
|
|
"max_ms": 850000,
|
|
"notes": "XXLarge repo: maximum production scale; sub-linear scaling expected"
|
|
}
|
|
},
|
|
"decomposition": {
|
|
"1000_files": {
|
|
"p50_ms": 4000,
|
|
"p95_ms": 10000,
|
|
"p99_ms": 16000,
|
|
"max_ms": 24000,
|
|
"notes": "Decomposition typically 2x indexing time"
|
|
},
|
|
"5000_files": {
|
|
"p50_ms": 20000,
|
|
"p95_ms": 50000,
|
|
"p99_ms": 80000,
|
|
"max_ms": 120000,
|
|
"notes": "Decomposition typically 2x indexing time"
|
|
},
|
|
"10000_files": {
|
|
"p50_ms": 40000,
|
|
"p95_ms": 100000,
|
|
"p99_ms": 160000,
|
|
"max_ms": 240000,
|
|
"notes": "Decomposition typically 2x indexing time"
|
|
},
|
|
"50000_files": {
|
|
"p50_ms": 160000,
|
|
"p95_ms": 400000,
|
|
"p99_ms": 640000,
|
|
"max_ms": 960000,
|
|
"notes": "Decomposition typically 2x indexing time; sub-linear scaling"
|
|
},
|
|
"100000_files": {
|
|
"p50_ms": 300000,
|
|
"p95_ms": 750000,
|
|
"p99_ms": 1200000,
|
|
"max_ms": 1800000,
|
|
"notes": "Decomposition typically 2x indexing time; sub-linear scaling"
|
|
}
|
|
},
|
|
"memory_usage_mb": {
|
|
"1000_files": {
|
|
"peak_mb": 256,
|
|
"steady_state_mb": 128
|
|
},
|
|
"5000_files": {
|
|
"peak_mb": 768,
|
|
"steady_state_mb": 384
|
|
},
|
|
"10000_files": {
|
|
"peak_mb": 1536,
|
|
"steady_state_mb": 768
|
|
},
|
|
"50000_files": {
|
|
"peak_mb": 6144,
|
|
"steady_state_mb": 3072
|
|
},
|
|
"100000_files": {
|
|
"peak_mb": 12288,
|
|
"steady_state_mb": 6144
|
|
}
|
|
},
|
|
"context_assembly": {
|
|
"100_fragments": {
|
|
"p50_ms": 50,
|
|
"p95_ms": 150,
|
|
"p99_ms": 300,
|
|
"notes": "Baseline context assembly at small scale"
|
|
},
|
|
"1000_fragments": {
|
|
"p50_ms": 500,
|
|
"p95_ms": 1500,
|
|
"p99_ms": 3000,
|
|
"notes": "Medium context assembly scale"
|
|
},
|
|
"5000_fragments": {
|
|
"p50_ms": 2500,
|
|
"p95_ms": 7500,
|
|
"p99_ms": 15000,
|
|
"notes": "Large context assembly; linear scaling expected"
|
|
},
|
|
"10000_fragments": {
|
|
"p50_ms": 5000,
|
|
"p95_ms": 15000,
|
|
"p99_ms": 30000,
|
|
"notes": "Maximum context assembly scale"
|
|
}
|
|
},
|
|
"execution_throughput": {
|
|
"10_plans": {
|
|
"p50_ms": 200,
|
|
"p95_ms": 500,
|
|
"p99_ms": 1000,
|
|
"notes": "Light execution workload"
|
|
},
|
|
"50_plans": {
|
|
"p50_ms": 1000,
|
|
"p95_ms": 2500,
|
|
"p99_ms": 5000,
|
|
"notes": "Medium execution workload"
|
|
},
|
|
"100_plans": {
|
|
"p50_ms": 2000,
|
|
"p95_ms": 5000,
|
|
"p99_ms": 10000,
|
|
"notes": "Heavy execution workload"
|
|
}
|
|
},
|
|
"scaling_expectations": {
|
|
"indexing_complexity": "O(n log n)",
|
|
"decomposition_complexity": "O(n log n)",
|
|
"context_assembly_complexity": "O(n log n)",
|
|
"execution_complexity": "O(n)",
|
|
"memory_complexity": "O(n)",
|
|
"notes": "Thresholds assume local SSD storage and 8+ GB RAM available"
|
|
}
|
|
}
|