b88bc0ec1b
CI / build (push) Successful in 19s
CI / lint (push) Successful in 3m19s
CI / quality (push) Successful in 3m43s
CI / typecheck (push) Successful in 3m55s
CI / security (push) Successful in 4m2s
CI / unit_tests (push) Successful in 6m39s
CI / integration_tests (push) Successful in 6m48s
CI / docker (push) Successful in 1m7s
CI / e2e_tests (push) Successful in 9m7s
CI / benchmark-regression (push) Has been skipped
CI / coverage (push) Failing after 16m36s
CI / benchmark-publish (push) Successful in 25m51s
CI / status-check (push) Failing after 4s
## Summary Add large project scaling benchmarks and tests at production scale (10K–100K files). ### New ASV Benchmarks **IndexingScalingSuite** (`large_project_scaling_bench.py`): - `time_walk_and_index` at 1K/10K/50K/100K files - `time_incremental_refresh` (1% modified files) - `track_indexed_file_count`, `track_tokens_per_second` **ContextAssemblyScalingSuite** (`context_assembly_scaling_bench.py`): - `time_full_pipeline` at 100/1K/5K/10K fragments - `time_tiered_strategy`, `time_recency_strategy` - `track_assembled_tokens`, `track_fragments_per_second` **ExecutionThroughputSuite** (`execution_throughput_bench.py`): - `time_sequential_plans` at 10/50/100 plans - `time_executor_construction`, `time_decision_tree_scaling` ### Scale Fixture Updates - Added `xlarge` (50K files) and `xxlarge` (100K files) profiles to `scale_metadata.json` - Added 50K/100K thresholds to `baseline_thresholds.json` - Added `context_assembly` and `execution_throughput` threshold sections ### Tests & Documentation - 15 Behave scenarios validating profiles, thresholds, monotonicity, memory budgets - 6 Robot integration tests including live 1K-file indexing throughput check - `docs/reference/scaling_baselines.md` documenting all baseline metrics ### Quality Gates | Session | Result | |---|---| | `nox -s lint` | PASS | | `nox -s typecheck` | PASS (0 errors) | | `nox -s unit_tests` | PASS (10,910 scenarios) | | `nox -s integration_tests` | PASS (1,526 tests) | | `nox -s coverage_report` | 97% (>= 97%) | Closes #859 Reviewed-on: #984 Co-authored-by: Brent E. Edwards <brent.edwards@cleverthis.com> Co-committed-by: Brent E. Edwards <brent.edwards@cleverthis.com>
144 lines
5.0 KiB
Python
144 lines
5.0 KiB
Python
"""ASV benchmarks for UKO Layer 3 technology vocabulary operations.
|
|
|
|
Measures the performance of:
|
|
- build_detail_level_map with insertions (Python: 3 insertions, 15-level map)
|
|
- build_detail_level_map without insertions (TS/RS/Java: identity, 12-level map)
|
|
- resolve_detail_level lookups (direct hit, parent fallback)
|
|
- UKOVocabulary construction (frozen Pydantic model with nested tuples)
|
|
- ProvenanceInfo construction
|
|
|
|
Based on specification.md Layer 3 Technology-Specific Specializations.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
from cleveragents.acms.uko.layer3_java import JAVA_DETAIL_LEVELS
|
|
from cleveragents.acms.uko.layer3_py import PYTHON_DETAIL_LEVELS
|
|
from cleveragents.acms.uko.layer3_rs import RUST_DETAIL_LEVELS
|
|
from cleveragents.acms.uko.layer3_ts import TYPESCRIPT_DETAIL_LEVELS
|
|
from cleveragents.acms.uko.vocabulary import (
|
|
OO_EFFECTIVE_LEVELS,
|
|
ProvenanceInfo,
|
|
UKOClass,
|
|
UKOVocabulary,
|
|
build_detail_level_map,
|
|
resolve_detail_level,
|
|
)
|
|
|
|
|
|
class BuildDetailLevelMapSuite:
|
|
"""Benchmark build_detail_level_map for Layer 3 maps."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare parent levels and insertions."""
|
|
self.oo_levels = OO_EFFECTIVE_LEVELS
|
|
self.py_insertions: tuple[tuple[str, int], ...] = (
|
|
("DECORATED_SIGNATURES", 6),
|
|
("TYPE_STUBS", 9),
|
|
("WITH_TESTS", 12),
|
|
)
|
|
self.empty_insertions: tuple[tuple[str, int], ...] = ()
|
|
|
|
def time_build_with_insertions(self) -> None:
|
|
"""Build Python 15-level map (3 insertions into 12 parents)."""
|
|
build_detail_level_map(self.oo_levels, self.py_insertions)
|
|
|
|
def time_build_identity(self) -> None:
|
|
"""Build identity map (no insertions -- TS/RS/Java pattern)."""
|
|
build_detail_level_map(self.oo_levels, self.empty_insertions)
|
|
|
|
def track_python_map_size(self) -> int:
|
|
"""Track number of levels in the Python effective map."""
|
|
return len(build_detail_level_map(self.oo_levels, self.py_insertions))
|
|
|
|
def track_identity_map_size(self) -> int:
|
|
"""Track number of levels in the identity map."""
|
|
return len(build_detail_level_map(self.oo_levels, self.empty_insertions))
|
|
|
|
|
|
class ResolveDetailLevelSuite:
|
|
"""Benchmark resolve_detail_level lookups."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare maps for resolution."""
|
|
self.py_levels = PYTHON_DETAIL_LEVELS
|
|
self.ts_levels = TYPESCRIPT_DETAIL_LEVELS
|
|
self.rs_levels = RUST_DETAIL_LEVELS
|
|
self.java_levels = JAVA_DETAIL_LEVELS
|
|
self.parent_levels = OO_EFFECTIVE_LEVELS
|
|
self.child_only_map: tuple[tuple[str, int], ...] = (("CHILD_ONLY", 0),)
|
|
|
|
def time_resolve_first_level(self) -> None:
|
|
"""Resolve MODULE_LISTING (first entry -- best case)."""
|
|
resolve_detail_level("MODULE_LISTING", self.py_levels)
|
|
|
|
def time_resolve_last_level(self) -> None:
|
|
"""Resolve WITH_TESTS (last entry -- worst case)."""
|
|
resolve_detail_level("WITH_TESTS", self.py_levels)
|
|
|
|
def time_resolve_inserted_level(self) -> None:
|
|
"""Resolve DECORATED_SIGNATURES (inserted level)."""
|
|
resolve_detail_level("DECORATED_SIGNATURES", self.py_levels)
|
|
|
|
def time_resolve_with_parent_fallback(self) -> None:
|
|
"""Resolve via parent fallback (name not in child map)."""
|
|
resolve_detail_level("MODULE_LISTING", self.child_only_map, self.parent_levels)
|
|
|
|
def time_resolve_ts(self) -> None:
|
|
"""Resolve SIGNATURES in TypeScript map."""
|
|
resolve_detail_level("SIGNATURES", self.ts_levels)
|
|
|
|
def time_resolve_rs(self) -> None:
|
|
"""Resolve SIGNATURES in Rust map."""
|
|
resolve_detail_level("SIGNATURES", self.rs_levels)
|
|
|
|
def time_resolve_java(self) -> None:
|
|
"""Resolve SIGNATURES in Java map."""
|
|
resolve_detail_level("SIGNATURES", self.java_levels)
|
|
|
|
|
|
class VocabularyConstructionSuite:
|
|
"""Benchmark UKOVocabulary and ProvenanceInfo construction."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
"""Pre-compute a fixed datetime to avoid syscall noise."""
|
|
self.fixed_datetime = datetime.now(tz=UTC)
|
|
|
|
def time_provenance_info(self) -> None:
|
|
"""Construct a ProvenanceInfo instance."""
|
|
ProvenanceInfo(
|
|
source_resource="res-01",
|
|
source_path="src/main.py",
|
|
valid_from=self.fixed_datetime,
|
|
)
|
|
|
|
def time_uko_class(self) -> None:
|
|
"""Construct a UKOClass instance."""
|
|
UKOClass(
|
|
uri="https://example.com/Test",
|
|
label="Test",
|
|
subclass_of=("https://example.com/Parent",),
|
|
)
|
|
|
|
def time_uko_vocabulary(self) -> None:
|
|
"""Construct a minimal UKOVocabulary instance."""
|
|
UKOVocabulary(
|
|
namespace="https://example.com/test#",
|
|
prefix="test:",
|
|
label="Test Vocabulary",
|
|
)
|