forked from cleveragents/cleveragents-core
brent.edwards
b88bc0ec1b
## Summary Add large project scaling benchmarks and tests at production scale (10K–100K files). ### New ASV Benchmarks **IndexingScalingSuite** (`large_project_scaling_bench.py`): - `time_walk_and_index` at 1K/10K/50K/100K files - `time_incremental_refresh` (1% modified files) - `track_indexed_file_count`, `track_tokens_per_second` **ContextAssemblyScalingSuite** (`context_assembly_scaling_bench.py`): - `time_full_pipeline` at 100/1K/5K/10K fragments - `time_tiered_strategy`, `time_recency_strategy` - `track_assembled_tokens`, `track_fragments_per_second` **ExecutionThroughputSuite** (`execution_throughput_bench.py`): - `time_sequential_plans` at 10/50/100 plans - `time_executor_construction`, `time_decision_tree_scaling` ### Scale Fixture Updates - Added `xlarge` (50K files) and `xxlarge` (100K files) profiles to `scale_metadata.json` - Added 50K/100K thresholds to `baseline_thresholds.json` - Added `context_assembly` and `execution_throughput` threshold sections ### Tests & Documentation - 15 Behave scenarios validating profiles, thresholds, monotonicity, memory budgets - 6 Robot integration tests including live 1K-file indexing throughput check - `docs/reference/scaling_baselines.md` documenting all baseline metrics ### Quality Gates | Session | Result | |---|---| | `nox -s lint` | PASS | | `nox -s typecheck` | PASS (0 errors) | | `nox -s unit_tests` | PASS (10,910 scenarios) | | `nox -s integration_tests` | PASS (1,526 tests) | | `nox -s coverage_report` | 97% (>= 97%) | Closes #859 Reviewed-on: cleveragents/cleveragents-core#984 Co-authored-by: Brent E. Edwards <brent.edwards@cleverthis.com> Co-committed-by: Brent E. Edwards <brent.edwards@cleverthis.com>
128 lines
2.8 KiB
Python
128 lines
2.8 KiB
Python
"""ASV benchmarks for async resource cleanup overhead (#321).
|
|
|
|
Measures registration, close_all, and leak-warning latency to establish
|
|
baselines for the AsyncResourceTracker.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
from cleveragents.core.async_cleanup import AsyncResourceTracker
|
|
|
|
|
|
class _FakeResource:
|
|
"""Minimal async resource for benchmarking."""
|
|
|
|
def __init__(self) -> None:
|
|
self.closed = False
|
|
|
|
async def close(self) -> None:
|
|
self.closed = True
|
|
|
|
|
|
def _make_tracker(count: int) -> AsyncResourceTracker:
|
|
"""Create a tracker pre-loaded with *count* fake resources."""
|
|
tracker = AsyncResourceTracker()
|
|
for i in range(count):
|
|
tracker.register(f"res-{i}", _FakeResource())
|
|
return tracker
|
|
|
|
|
|
class TimeRegisterSingle:
|
|
"""Benchmark registering a single resource."""
|
|
|
|
timeout = 10
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = AsyncResourceTracker()
|
|
self.counter = 0
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_register_one(self) -> None:
|
|
name = f"bench-{self.counter}"
|
|
self.counter += 1
|
|
self.tracker.register(name, _FakeResource())
|
|
|
|
|
|
class TimeRegisterBatch:
|
|
"""Benchmark registering 100 resources in sequence."""
|
|
|
|
timeout = 10
|
|
number = 1 # Tracker cannot re-register names; recreate each iteration.
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = AsyncResourceTracker()
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_register_100(self) -> None:
|
|
tracker = AsyncResourceTracker()
|
|
for i in range(100):
|
|
tracker.register(f"batch-{i}", _FakeResource())
|
|
|
|
|
|
class TimeCloseAll:
|
|
"""Benchmark close_all on pre-loaded trackers."""
|
|
|
|
timeout = 30
|
|
number = 1 # Re-create tracker for each iteration.
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = _make_tracker(50)
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_close_50_resources(self) -> None:
|
|
asyncio.run(self.tracker.close_all())
|
|
|
|
|
|
class TimeCloseAllLarge:
|
|
"""Benchmark close_all with 500 resources."""
|
|
|
|
timeout = 60
|
|
number = 1
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = _make_tracker(500)
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_close_500_resources(self) -> None:
|
|
asyncio.run(self.tracker.close_all())
|
|
|
|
|
|
class TimeLeakWarning:
|
|
"""Benchmark the _warn_unclosed finalizer path."""
|
|
|
|
timeout = 10
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = _make_tracker(20)
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_warn_20_unclosed(self) -> None:
|
|
self.tracker._warn_unclosed()
|
|
|
|
|
|
class TimeOpenCount:
|
|
"""Benchmark the open_count property."""
|
|
|
|
timeout = 10
|
|
|
|
def setup(self) -> None:
|
|
self.tracker = _make_tracker(100)
|
|
|
|
def teardown(self) -> None:
|
|
pass
|
|
|
|
def time_open_count(self) -> None:
|
|
_ = self.tracker.open_count
|