Files
cleveragents-core/benchmarks/bench_uko_layer3.py
T
brent.edwards b88bc0ec1b
CI / build (push) Successful in 19s
CI / lint (push) Successful in 3m19s
CI / quality (push) Successful in 3m43s
CI / typecheck (push) Successful in 3m55s
CI / security (push) Successful in 4m2s
CI / unit_tests (push) Successful in 6m39s
CI / integration_tests (push) Successful in 6m48s
CI / docker (push) Successful in 1m7s
CI / e2e_tests (push) Successful in 9m7s
CI / benchmark-regression (push) Has been skipped
CI / coverage (push) Failing after 16m36s
CI / benchmark-publish (push) Successful in 25m51s
CI / status-check (push) Failing after 4s
feat(perf): large project scaling tests (#984)
## Summary

Add large project scaling benchmarks and tests at production scale (10K–100K files).

### New ASV Benchmarks

**IndexingScalingSuite** (`large_project_scaling_bench.py`):
- `time_walk_and_index` at 1K/10K/50K/100K files
- `time_incremental_refresh` (1% modified files)
- `track_indexed_file_count`, `track_tokens_per_second`

**ContextAssemblyScalingSuite** (`context_assembly_scaling_bench.py`):
- `time_full_pipeline` at 100/1K/5K/10K fragments
- `time_tiered_strategy`, `time_recency_strategy`
- `track_assembled_tokens`, `track_fragments_per_second`

**ExecutionThroughputSuite** (`execution_throughput_bench.py`):
- `time_sequential_plans` at 10/50/100 plans
- `time_executor_construction`, `time_decision_tree_scaling`

### Scale Fixture Updates

- Added `xlarge` (50K files) and `xxlarge` (100K files) profiles to `scale_metadata.json`
- Added 50K/100K thresholds to `baseline_thresholds.json`
- Added `context_assembly` and `execution_throughput` threshold sections

### Tests & Documentation

- 15 Behave scenarios validating profiles, thresholds, monotonicity, memory budgets
- 6 Robot integration tests including live 1K-file indexing throughput check
- `docs/reference/scaling_baselines.md` documenting all baseline metrics

### Quality Gates

| Session | Result |
|---|---|
| `nox -s lint` | PASS |
| `nox -s typecheck` | PASS (0 errors) |
| `nox -s unit_tests` | PASS (10,910 scenarios) |
| `nox -s integration_tests` | PASS (1,526 tests) |
| `nox -s coverage_report` | 97% (>= 97%) |

Closes #859

Reviewed-on: #984
Co-authored-by: Brent E. Edwards <brent.edwards@cleverthis.com>
Co-committed-by: Brent E. Edwards <brent.edwards@cleverthis.com>
2026-03-21 04:46:45 +00:00

144 lines
5.0 KiB
Python

"""ASV benchmarks for UKO Layer 3 technology vocabulary operations.
Measures the performance of:
- build_detail_level_map with insertions (Python: 3 insertions, 15-level map)
- build_detail_level_map without insertions (TS/RS/Java: identity, 12-level map)
- resolve_detail_level lookups (direct hit, parent fallback)
- UKOVocabulary construction (frozen Pydantic model with nested tuples)
- ProvenanceInfo construction
Based on specification.md Layer 3 Technology-Specific Specializations.
"""
from __future__ import annotations
import sys
from datetime import UTC, datetime
from pathlib import Path
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.acms.uko.layer3_java import JAVA_DETAIL_LEVELS
from cleveragents.acms.uko.layer3_py import PYTHON_DETAIL_LEVELS
from cleveragents.acms.uko.layer3_rs import RUST_DETAIL_LEVELS
from cleveragents.acms.uko.layer3_ts import TYPESCRIPT_DETAIL_LEVELS
from cleveragents.acms.uko.vocabulary import (
OO_EFFECTIVE_LEVELS,
ProvenanceInfo,
UKOClass,
UKOVocabulary,
build_detail_level_map,
resolve_detail_level,
)
class BuildDetailLevelMapSuite:
"""Benchmark build_detail_level_map for Layer 3 maps."""
timeout = 60
def setup(self) -> None:
"""Prepare parent levels and insertions."""
self.oo_levels = OO_EFFECTIVE_LEVELS
self.py_insertions: tuple[tuple[str, int], ...] = (
("DECORATED_SIGNATURES", 6),
("TYPE_STUBS", 9),
("WITH_TESTS", 12),
)
self.empty_insertions: tuple[tuple[str, int], ...] = ()
def time_build_with_insertions(self) -> None:
"""Build Python 15-level map (3 insertions into 12 parents)."""
build_detail_level_map(self.oo_levels, self.py_insertions)
def time_build_identity(self) -> None:
"""Build identity map (no insertions -- TS/RS/Java pattern)."""
build_detail_level_map(self.oo_levels, self.empty_insertions)
def track_python_map_size(self) -> int:
"""Track number of levels in the Python effective map."""
return len(build_detail_level_map(self.oo_levels, self.py_insertions))
def track_identity_map_size(self) -> int:
"""Track number of levels in the identity map."""
return len(build_detail_level_map(self.oo_levels, self.empty_insertions))
class ResolveDetailLevelSuite:
"""Benchmark resolve_detail_level lookups."""
timeout = 60
def setup(self) -> None:
"""Prepare maps for resolution."""
self.py_levels = PYTHON_DETAIL_LEVELS
self.ts_levels = TYPESCRIPT_DETAIL_LEVELS
self.rs_levels = RUST_DETAIL_LEVELS
self.java_levels = JAVA_DETAIL_LEVELS
self.parent_levels = OO_EFFECTIVE_LEVELS
self.child_only_map: tuple[tuple[str, int], ...] = (("CHILD_ONLY", 0),)
def time_resolve_first_level(self) -> None:
"""Resolve MODULE_LISTING (first entry -- best case)."""
resolve_detail_level("MODULE_LISTING", self.py_levels)
def time_resolve_last_level(self) -> None:
"""Resolve WITH_TESTS (last entry -- worst case)."""
resolve_detail_level("WITH_TESTS", self.py_levels)
def time_resolve_inserted_level(self) -> None:
"""Resolve DECORATED_SIGNATURES (inserted level)."""
resolve_detail_level("DECORATED_SIGNATURES", self.py_levels)
def time_resolve_with_parent_fallback(self) -> None:
"""Resolve via parent fallback (name not in child map)."""
resolve_detail_level("MODULE_LISTING", self.child_only_map, self.parent_levels)
def time_resolve_ts(self) -> None:
"""Resolve SIGNATURES in TypeScript map."""
resolve_detail_level("SIGNATURES", self.ts_levels)
def time_resolve_rs(self) -> None:
"""Resolve SIGNATURES in Rust map."""
resolve_detail_level("SIGNATURES", self.rs_levels)
def time_resolve_java(self) -> None:
"""Resolve SIGNATURES in Java map."""
resolve_detail_level("SIGNATURES", self.java_levels)
class VocabularyConstructionSuite:
"""Benchmark UKOVocabulary and ProvenanceInfo construction."""
timeout = 60
def setup(self) -> None:
"""Pre-compute a fixed datetime to avoid syscall noise."""
self.fixed_datetime = datetime.now(tz=UTC)
def time_provenance_info(self) -> None:
"""Construct a ProvenanceInfo instance."""
ProvenanceInfo(
source_resource="res-01",
source_path="src/main.py",
valid_from=self.fixed_datetime,
)
def time_uko_class(self) -> None:
"""Construct a UKOClass instance."""
UKOClass(
uri="https://example.com/Test",
label="Test",
subclass_of=("https://example.com/Parent",),
)
def time_uko_vocabulary(self) -> None:
"""Construct a minimal UKOVocabulary instance."""
UKOVocabulary(
namespace="https://example.com/test#",
prefix="test:",
label="Test Vocabulary",
)