cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

test(perf): add scale test fixtures

Browse Source
This commit is contained in:
Brent Edwards
2026-02-20 02:45:20 +00:00
parent 4243ac8cff
commit 0dfdc02564
12 changed files with 1333 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/scale_fixture_bench.py
+106
View File
@@ -0,0 +1,106 @@
"""ASV benchmarks for scale fixture processing.
Measures the performance of:
- Fixture metadata loading and parsing
- Threshold matrix validation
- File distribution generation from scale profiles
- Language mix validation
"""
from __future__ import annotations
import json
import math
from pathlib import Path
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "scale"
def _load_fixture(filename: str) -> dict:
"""Load a JSON fixture file from the scale fixtures directory."""
path = _FIXTURES_DIR / filename
with open(path) as fh:
return json.load(fh)
class ScaleFixtureSuite:
"""Benchmarks for scale fixture processing."""
timeout = 120
def setup(self) -> None:
"""Load scale fixtures for benchmark use."""
self.metadata_path = _FIXTURES_DIR / "scale_metadata.json"
self.thresholds_path = _FIXTURES_DIR / "baseline_thresholds.json"
self.metadata = _load_fixture("scale_metadata.json")
self.thresholds = _load_fixture("baseline_thresholds.json")
def time_metadata_loading(self) -> None:
"""Benchmark fixture metadata loading from disk."""
_load_fixture("scale_metadata.json")
def time_threshold_loading(self) -> None:
"""Benchmark threshold fixture loading from disk."""
_load_fixture("baseline_thresholds.json")
def time_threshold_validation(self) -> None:
"""Benchmark threshold matrix validation logic."""
data = self.thresholds
for section in ("indexing", "decomposition"):
for _file_count, thresholds in data[section].items():
p50 = thresholds["p50_ms"]
p95 = thresholds["p95_ms"]
p99 = thresholds["p99_ms"]
assert p50 < p95 < p99
def time_language_mix_validation(self) -> None:
"""Benchmark language mix sum validation across all profiles."""
for profile in self.metadata["scale_profiles"]:
total = sum(profile["language_mix"].values())
assert math.isclose(total, 1.0, rel_tol=1e-6)
def time_file_distribution_generation(self) -> None:
"""Benchmark file distribution generation for all profiles."""
for profile in self.metadata["scale_profiles"]:
total = profile["file_count"]
mix = profile["language_mix"]
distribution: dict[str, int] = {}
allocated = 0
items = sorted(mix.items(), key=lambda x: x[1], reverse=True)
for i, (lang, ratio) in enumerate(items):
if i == len(items) - 1:
distribution[lang] = total - allocated
else:
count = round(total * ratio)
distribution[lang] = count
allocated += count
assert sum(distribution.values()) == total
def time_full_fixture_roundtrip(self) -> None:
"""Benchmark complete fixture load + validate + distribute cycle."""
metadata = _load_fixture("scale_metadata.json")
thresholds = _load_fixture("baseline_thresholds.json")
# Validate thresholds
for section in ("indexing", "decomposition"):
for _file_count, thresh in thresholds[section].items():
assert thresh["p50_ms"] < thresh["p95_ms"] < thresh["p99_ms"]
# Validate and generate distributions
for profile in metadata["scale_profiles"]:
total_mix = sum(profile["language_mix"].values())
assert math.isclose(total_mix, 1.0, rel_tol=1e-6)
total = profile["file_count"]
mix = profile["language_mix"]
distribution: dict[str, int] = {}
allocated = 0
items = sorted(mix.items(), key=lambda x: x[1], reverse=True)
for i, (lang, ratio) in enumerate(items):
if i == len(items) - 1:
distribution[lang] = total - allocated
else:
count = round(total * ratio)
distribution[lang] = count
allocated += count
assert sum(distribution.values()) == total
docs/development/scale_testing.md
+168
View File
@@ -0,0 +1,168 @@
# Scale Testing Runbook
## Overview
Scale testing validates that CleverAgents performs within acceptable bounds when
indexing and decomposing repositories of varying sizes (1K, 5K, and 10K files).
The test suite uses pre-defined fixtures rather than live repositories to ensure
repeatable, deterministic results.
### Approach
1. **Fixture-based**: Scale profiles are defined in JSON metadata files that
describe repository characteristics (file count, size, language mix) without
requiring actual large repositories.
2. **Threshold matrices**: Baseline performance thresholds are maintained in a
separate JSON file with p50/p95/p99 percentile targets for indexing and
decomposition operations.
3. **Simulation**: File distribution generation simulates how files would be
allocated across languages for each profile, validating the distribution
logic without creating actual files.
## Environment Requirements
- Python 3.13+
- At least 8 GB RAM (for large profile validation)
- Local SSD storage recommended
- All dependencies installed via `uv sync`
## Fixture Files
All scale fixtures live in `features/fixtures/scale/`:
| File | Purpose |
|------|---------|
| `scale_metadata.json` | Scale profile definitions (file count, size, language mix) |
| `baseline_thresholds.json` | Performance threshold matrices (p50/p95/p99 per operation) |
| `generator_instructions.md` | Manual instructions for generating synthetic test repos |
## Running Scale Tests
### Behave (Unit Tests)
```bash
# Run scale test scenarios only
nox -s unit_tests -- features/scale_test.feature
# Run all unit tests including scale tests
nox -s unit_tests
```
### Robot Framework (Integration Tests)
```bash
# Run scale test integration suite only
nox -s integration_tests -- --suite robot/scale_test.robot
# Run all integration tests including scale tests
nox -s integration_tests
```
### ASV Benchmarks
```bash
# Run all benchmarks including scale fixture benchmarks
nox -s benchmark
```
## Interpreting Results
### Behave Results
All 20 scenarios should pass. Failures indicate:
- **Fixture loading failures**: Missing or malformed JSON files
- **Threshold validation failures**: Percentile ordering violations (p50 >= p95)
- **Language mix failures**: Distribution ratios don't sum to 1.0
- **Distribution generation failures**: File count allocation doesn't match profile
### Robot Results
All 6 test cases should pass. Each test prints a sentinel string on success:
| Sentinel | Meaning |
|----------|---------|
| `scale-metadata-ok` | Metadata fixture loaded and has valid structure |
| `scale-profiles-ok` | All profiles have required fields and valid language mixes |
| `scale-thresholds-ok` | Threshold matrix has all required sections and percentiles |
| `scale-monotonicity-ok` | All thresholds satisfy p50 < p95 < p99 |
| `scale-distribution-ok` | File distributions match language mix ratios |
| `scale-generator-docs-ok` | Generator instructions document is present |
### ASV Benchmarks
The `ScaleFixtureSuite` measures:
| Benchmark | Description | Expected Range |
|-----------|-------------|---------------|
| `time_metadata_loading` | JSON parse time for metadata | < 1 ms |
| `time_threshold_loading` | JSON parse time for thresholds | < 1 ms |
| `time_threshold_validation` | Threshold matrix validation | < 0.1 ms |
| `time_language_mix_validation` | Language mix sum checks | < 0.1 ms |
| `time_file_distribution_generation` | Distribution calculation | < 0.5 ms |
| `time_full_fixture_roundtrip` | Complete load + validate + distribute | < 2 ms |
## Threshold Adjustment Guidelines
### When to Adjust Thresholds
- After significant indexing algorithm changes
- When targeting new hardware profiles (e.g., CI runners with different specs)
- When adding support for new file types that affect parsing performance
- After profiling reveals consistent threshold violations
### How to Adjust
1. Run benchmarks on the target environment:
```bash
nox -s benchmark
```
2. Collect actual p50/p95/p99 values from multiple runs (minimum 10 runs recommended).
3. Update `features/fixtures/scale/baseline_thresholds.json` with new values.
4. Ensure the monotonicity invariant holds: `p50 < p95 < p99` for all entries.
5. Update the `scaling_expectations` section if the algorithmic complexity has changed.
6. Run the full test suite to verify:
```bash
nox -s unit_tests -- features/scale_test.feature
nox -s integration_tests -- --suite robot/scale_test.robot
```
### Threshold Safety Margins
Current thresholds include a 2x safety margin over measured values to account for:
- CI runner variability
- Concurrent workload interference
- OS-level caching effects
- GC pauses in the Python runtime
When adjusting, maintain at least a 1.5x margin over observed p99 values.
## Adding New Scale Profiles
To add a new scale profile (e.g., "extra-large" for 50K files):
1. Add the profile entry to `scale_metadata.json`:
```json
{
"name": "extra-large",
"file_count": 50000,
"total_size_mb": 2500,
"language_mix": { ... },
"expected_index_time_s": 250.0,
"expected_decomposition_time_s": 500.0,
"description": "Extra-large enterprise monorepo"
}
```
2. Add threshold entries to `baseline_thresholds.json` for `"50000_files"`.
3. Update Behave scenarios in `features/scale_test.feature` if exact profile
count assertions need updating.
4. Run tests to verify the new profile integrates correctly.
docs/development/testing.md
+36
View File
@@ -438,3 +438,39 @@ Add `timeout` parameters to `Run Process` calls in Robot tests:
```robot
${result}= Run Process ${PYTHON} -m cleveragents ... timeout=30s
```
## Scale Testing
Scale tests validate performance baselines for repository indexing and
decomposition across different repository sizes (1K, 5K, and 10K files).
### Fixtures
Scale test fixtures live in `features/fixtures/scale/`:
- `scale_metadata.json` — Scale profile definitions (file count, language mix, expected timings)
- `baseline_thresholds.json` — Performance threshold matrices (p50/p95/p99 per operation per file count)
- `generator_instructions.md` — Manual instructions for generating synthetic repos
### Running Scale Tests
```bash
# Behave scale test scenarios
nox -s unit_tests -- features/scale_test.feature
# Robot scale test integration
nox -s integration_tests -- --suite robot/scale_test.robot
# ASV scale fixture benchmarks
nox -s benchmark
```
### Scale Test Suites
| Suite | Framework | Scenarios | Description |
|-------|-----------|-----------|-------------|
| `features/scale_test.feature` | Behave | 20 | Fixture loading, metadata validation, threshold checks, distribution simulation |
| `robot/scale_test.robot` | Robot | 6 | Integration-level fixture validation via helper script |
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
For full scale testing documentation, see `docs/development/scale_testing.md`.
features/fixtures/scale/baseline_thresholds.json
+70
View File
@@ -0,0 +1,70 @@
{
"schema_version": "1.0",
"description": "Baseline performance thresholds for scale testing. Values are in milliseconds unless noted otherwise.",
"indexing": {
"1000_files": {
"p50_ms": 2000,
"p95_ms": 5000,
"p99_ms": 8000,
"max_ms": 12000,
"notes": "Small repo: single microservice scale"
},
"5000_files": {
"p50_ms": 10000,
"p95_ms": 25000,
"p99_ms": 40000,
"max_ms": 60000,
"notes": "Medium repo: multi-service monorepo scale"
},
"10000_files": {
"p50_ms": 18000,
"p95_ms": 45000,
"p99_ms": 72000,
"max_ms": 110000,
"notes": "Large repo: enterprise monorepo scale; sub-linear scaling expected"
}
},
"decomposition": {
"1000_files": {
"p50_ms": 4000,
"p95_ms": 10000,
"p99_ms": 16000,
"max_ms": 24000,
"notes": "Decomposition typically 2x indexing time"
},
"5000_files": {
"p50_ms": 20000,
"p95_ms": 50000,
"p99_ms": 80000,
"max_ms": 120000,
"notes": "Decomposition typically 2x indexing time"
},
"10000_files": {
"p50_ms": 40000,
"p95_ms": 100000,
"p99_ms": 160000,
"max_ms": 240000,
"notes": "Decomposition typically 2x indexing time"
}
},
"memory_usage_mb": {
"1000_files": {
"peak_mb": 256,
"steady_state_mb": 128
},
"5000_files": {
"peak_mb": 768,
"steady_state_mb": 384
},
"10000_files": {
"peak_mb": 1536,
"steady_state_mb": 768
}
},
"scaling_expectations": {
"indexing_complexity": "O(n log n)",
"decomposition_complexity": "O(n log n)",
"memory_complexity": "O(n)",
"notes": "Thresholds assume local SSD storage and 8+ GB RAM available"
}
}
features/fixtures/scale/generator_instructions.md
+165
View File
@@ -0,0 +1,165 @@
# Scale Fixture Generator Instructions
## Overview
This document describes how to generate synthetic repositories for scale testing
without requiring helper scripts. The process is entirely manual and repeatable
using standard Unix tools and Python standard library modules.
## Prerequisites
- Python 3.13+
- Standard Unix tools: `mkdir`, `cp`, `find`
- At least 2 GB free disk space for large profile generation
## Generating a Small Profile (1,000 files)
### Step 1: Create the directory structure
```bash
mkdir -p /tmp/scale-test-repo/src/{api,core,models,utils,tests}
mkdir -p /tmp/scale-test-repo/config
mkdir -p /tmp/scale-test-repo/scripts
```
### Step 2: Generate Python files (60% = 600 files)
Create a template Python file and replicate it with unique module names:
```bash
for i in $(seq 1 600); do
dir="src/$(echo api core models utils tests | tr ' ' '\n' | shuf -n1)"
cat > "/tmp/scale-test-repo/${dir}/module_${i}.py" << 'EOF'
"""Auto-generated module for scale testing."""
class ScaleTestClass:
"""Placeholder class for indexing benchmarks."""
def __init__(self, name: str) -> None:
self.name = name
def process(self) -> str:
return f"processed-{self.name}"
EOF
done
```
### Step 3: Generate JavaScript files (30% = 300 files)
```bash
for i in $(seq 1 300); do
dir="src/$(echo api core utils | tr ' ' '\n' | shuf -n1)"
cat > "/tmp/scale-test-repo/${dir}/module_${i}.js" << 'EOF'
// Auto-generated module for scale testing
export class ScaleTestClass {
constructor(name) {
this.name = name;
}
process() {
return `processed-${this.name}`;
}
}
EOF
done
```
### Step 4: Generate YAML files (10% = 100 files)
```bash
for i in $(seq 1 100); do
cat > "/tmp/scale-test-repo/config/config_${i}.yaml" << EOF
name: scale-config-${i}
version: "1.0"
settings:
enabled: true
timeout: 30
retries: 3
EOF
done
```
## Generating a Medium Profile (5,000 files)
Follow the same steps but multiply file counts by 5:
- Python: 2,500 files
- JavaScript: 1,250 files
- TypeScript: 750 files
- YAML: 500 files
Add TypeScript generation:
```bash
for i in $(seq 1 750); do
dir="src/$(echo api core models | tr ' ' '\n' | shuf -n1)"
cat > "/tmp/scale-test-repo/${dir}/module_${i}.ts" << 'EOF'
// Auto-generated TypeScript module for scale testing
export interface ScaleTestData {
name: string;
value: number;
}
export function process(data: ScaleTestData): string {
return `processed-${data.name}`;
}
EOF
done
```
## Generating a Large Profile (10,000 files)
Follow the same steps but multiply file counts by 10:
- Python: 4,000 files
- JavaScript: 2,000 files
- TypeScript: 2,000 files
- YAML: 1,000 files
- JSON: 1,000 files
Add JSON generation:
```bash
for i in $(seq 1 1000); do
cat > "/tmp/scale-test-repo/config/data_${i}.json" << EOF
{
"name": "scale-data-${i}",
"version": "1.0",
"entries": [
{"key": "item_1", "value": ${i}},
{"key": "item_2", "value": $((i * 2))}
]
}
EOF
done
```
## Verification
After generation, verify the fixture matches the expected profile:
```bash
# Count total files
find /tmp/scale-test-repo -type f | wc -l
# Count by extension
find /tmp/scale-test-repo -name "*.py" | wc -l
find /tmp/scale-test-repo -name "*.js" | wc -l
find /tmp/scale-test-repo -name "*.ts" | wc -l
find /tmp/scale-test-repo -name "*.yaml" | wc -l
find /tmp/scale-test-repo -name "*.json" | wc -l
# Check total size
du -sh /tmp/scale-test-repo
```
## Cleanup
```bash
rm -rf /tmp/scale-test-repo
```
## Notes
- Generated files are intentionally simple to isolate indexing and decomposition
overhead from parsing complexity.
- Language mix ratios should match the `scale_metadata.json` profiles.
- For reproducible benchmarks, always generate fresh fixtures before each run.
- No external dependencies or helper scripts are required.
features/fixtures/scale/scale_metadata.json
+72
View File
@@ -0,0 +1,72 @@
{
"schema_version": "1.0",
"description": "Scale test fixture metadata for CleverAgents performance testing",
"scale_profiles": [
{
"name": "small",
"file_count": 1000,
"total_size_mb": 50,
"language_mix": {
"python": 0.6,
"javascript": 0.3,
"yaml": 0.1
},
"expected_index_time_s": 5.0,
"expected_decomposition_time_s": 10.0,
"description": "Small repository representative of a single microservice"
},
{
"name": "medium",
"file_count": 5000,
"total_size_mb": 250,
"language_mix": {
"python": 0.5,
"javascript": 0.25,
"typescript": 0.15,
"yaml": 0.1
},
"expected_index_time_s": 25.0,
"expected_decomposition_time_s": 50.0,
"description": "Medium repository representative of a monorepo with several services"
},
{
"name": "large",
"file_count": 10000,
"total_size_mb": 500,
"language_mix": {
"python": 0.4,
"javascript": 0.2,
"typescript": 0.2,
"yaml": 0.1,
"json": 0.1
},
"expected_index_time_s": 50.0,
"expected_decomposition_time_s": 100.0,
"description": "Large repository representative of a full enterprise monorepo"
}
],
"supported_languages": [
"python",
"javascript",
"typescript",
"yaml",
"json",
"markdown",
"html",
"css",
"sql",
"shell"
],
"avg_file_size_kb": {
"python": 8.5,
"javascript": 6.2,
"typescript": 7.8,
"yaml": 2.1,
"json": 3.4,
"markdown": 4.0,
"html": 12.0,
"css": 5.5,
"sql": 3.0,
"shell": 1.5
}
}
features/scale_test.feature
+122
View File
@@ -0,0 +1,122 @@
Feature: Scale test fixtures and performance baselines
As a performance engineer
I want to validate scale test fixtures and threshold matrices
So that performance baselines are well-formed and realistic
Background:
Given a scale test environment is ready
# ──────────────────────────────────────────────────
# Section 1: Fixture loading and metadata validation
# ──────────────────────────────────────────────────
Scenario: Load scale metadata fixture from disk
When I scale test load the scale metadata fixture
Then the scale test metadata should contain a schema version
And the scale test metadata should contain scale profiles
Scenario: Scale metadata contains all required profiles
When I scale test load the scale metadata fixture
Then the scale test metadata should have exactly 3 profiles
And the scale test profile names should be "small, medium, large"
Scenario: Each scale profile has required fields
When I scale test load the scale metadata fixture
Then each scale test profile should have a file count
And each scale test profile should have a total size in megabytes
And each scale test profile should have a language mix
And each scale test profile should have expected index time
And each scale test profile should have expected decomposition time
Scenario: Scale metadata lists supported languages
When I scale test load the scale metadata fixture
Then the scale test metadata should list supported languages
And the scale test supported languages should include "python"
And the scale test supported languages should include "javascript"
# ──────────────────────────────────────────────────
# Section 2: Threshold matrix validation
# ──────────────────────────────────────────────────
Scenario: Load baseline thresholds fixture from disk
When I scale test load the baseline thresholds fixture
Then the scale test thresholds should contain a schema version
And the scale test thresholds should have indexing section
And the scale test thresholds should have decomposition section
Scenario: Indexing thresholds cover all file counts
When I scale test load the baseline thresholds fixture
Then the scale test indexing thresholds should cover "1000_files"
And the scale test indexing thresholds should cover "5000_files"
And the scale test indexing thresholds should cover "10000_files"
Scenario: Each indexing threshold has percentile fields
When I scale test load the baseline thresholds fixture
Then each scale test indexing threshold should have p50 p95 and p99
Scenario: Decomposition thresholds cover all file counts
When I scale test load the baseline thresholds fixture
Then the scale test decomposition thresholds should cover "1000_files"
And the scale test decomposition thresholds should cover "5000_files"
And the scale test decomposition thresholds should cover "10000_files"
Scenario: Each decomposition threshold has percentile fields
When I scale test load the baseline thresholds fixture
Then each scale test decomposition threshold should have p50 p95 and p99
Scenario: Thresholds are monotonically increasing across percentiles
When I scale test load the baseline thresholds fixture
Then scale test indexing p50 should be less than p95 for all file counts
And scale test indexing p95 should be less than p99 for all file counts
# ──────────────────────────────────────────────────
# Section 3: Language mix distribution validation
# ──────────────────────────────────────────────────
Scenario: Language mix sums to 1.0 for each profile
When I scale test load the scale metadata fixture
Then the scale test language mix should sum to 1.0 for each profile
Scenario: Language mix values are between 0 and 1
When I scale test load the scale metadata fixture
Then each scale test language mix value should be between 0 and 1
Scenario: All languages in mix are in supported languages list
When I scale test load the scale metadata fixture
Then all scale test language mix keys should be in supported languages
# ──────────────────────────────────────────────────
# Section 4: Scale profile simulation
# ──────────────────────────────────────────────────
Scenario: Generate file count distribution for small profile
When I scale test generate file distribution for the "small" profile
Then the scale test total generated file count should be 1000
And the scale test generated distribution should match the language mix
Scenario: Generate file count distribution for medium profile
When I scale test generate file distribution for the "medium" profile
Then the scale test total generated file count should be 5000
And the scale test generated distribution should match the language mix
Scenario: Generate file count distribution for large profile
When I scale test generate file distribution for the "large" profile
Then the scale test total generated file count should be 10000
And the scale test generated distribution should match the language mix
# ──────────────────────────────────────────────────
# Section 5: Baseline threshold realism checks
# ──────────────────────────────────────────────────
Scenario: Indexing thresholds scale sub-linearly with file count
When I scale test load the baseline thresholds fixture
Then scale test 10K indexing p50 should be less than 10x the 1K p50
Scenario: Decomposition thresholds exceed indexing thresholds
When I scale test load the baseline thresholds fixture
Then scale test decomposition p50 should be greater than indexing p50 for each file count
Scenario: Memory thresholds are present and realistic
When I scale test load the baseline thresholds fixture
Then the scale test thresholds should have memory usage section
And scale test memory peak should exceed steady state for each file count
features/steps/scale_test_steps.py
+330
View File
@@ -0,0 +1,330 @@
"""Step definitions for scale test fixture validation."""
from __future__ import annotations
import json
import math
from pathlib import Path
from behave import given, then, when
from behave.runner import Context
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "scale"
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a scale test environment is ready")
def step_scale_test_environment_ready(context: Context) -> None:
"""Verify scale test fixtures directory exists."""
assert _FIXTURES_DIR.is_dir(), f"Fixtures dir missing: {_FIXTURES_DIR}"
context.scale_metadata = None
context.scale_thresholds = None
context.scale_file_distribution = None
# ---------------------------------------------------------------------------
# Section 1: Fixture loading and metadata validation
# ---------------------------------------------------------------------------
@when("I scale test load the scale metadata fixture")
def step_scale_test_load_metadata(context: Context) -> None:
"""Load scale_metadata.json from fixtures."""
path = _FIXTURES_DIR / "scale_metadata.json"
assert path.exists(), f"Missing fixture: {path}"
with open(path) as fh:
context.scale_metadata = json.load(fh)
@then("the scale test metadata should contain a schema version")
def step_scale_test_metadata_has_schema_version(context: Context) -> None:
assert "schema_version" in context.scale_metadata
@then("the scale test metadata should contain scale profiles")
def step_scale_test_metadata_has_profiles(context: Context) -> None:
assert "scale_profiles" in context.scale_metadata
assert len(context.scale_metadata["scale_profiles"]) > 0
@then("the scale test metadata should have exactly {count:d} profiles")
def step_scale_test_metadata_profile_count(context: Context, count: int) -> None:
assert len(context.scale_metadata["scale_profiles"]) == count
@then('the scale test profile names should be "{names}"')
def step_scale_test_profile_names(context: Context, names: str) -> None:
expected = [n.strip() for n in names.split(",")]
actual = [p["name"] for p in context.scale_metadata["scale_profiles"]]
assert actual == expected, f"Expected {expected}, got {actual}"
@then("each scale test profile should have a file count")
def step_scale_test_profile_has_file_count(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
assert "file_count" in profile, f"Missing file_count in {profile['name']}"
assert isinstance(profile["file_count"], int)
@then("each scale test profile should have a total size in megabytes")
def step_scale_test_profile_has_total_size(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
assert "total_size_mb" in profile, f"Missing total_size_mb in {profile['name']}"
assert isinstance(profile["total_size_mb"], (int, float))
@then("each scale test profile should have a language mix")
def step_scale_test_profile_has_language_mix(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
assert "language_mix" in profile, f"Missing language_mix in {profile['name']}"
assert isinstance(profile["language_mix"], dict)
@then("each scale test profile should have expected index time")
def step_scale_test_profile_has_index_time(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
assert "expected_index_time_s" in profile, (
f"Missing expected_index_time_s in {profile['name']}"
)
@then("each scale test profile should have expected decomposition time")
def step_scale_test_profile_has_decomposition_time(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
assert "expected_decomposition_time_s" in profile, (
f"Missing expected_decomposition_time_s in {profile['name']}"
)
@then("the scale test metadata should list supported languages")
def step_scale_test_metadata_has_supported_languages(context: Context) -> None:
assert "supported_languages" in context.scale_metadata
assert len(context.scale_metadata["supported_languages"]) > 0
@then('the scale test supported languages should include "{language}"')
def step_scale_test_supported_languages_include(
context: Context, language: str
) -> None:
assert language in context.scale_metadata["supported_languages"]
# ---------------------------------------------------------------------------
# Section 2: Threshold matrix validation
# ---------------------------------------------------------------------------
@when("I scale test load the baseline thresholds fixture")
def step_scale_test_load_thresholds(context: Context) -> None:
"""Load baseline_thresholds.json from fixtures."""
path = _FIXTURES_DIR / "baseline_thresholds.json"
assert path.exists(), f"Missing fixture: {path}"
with open(path) as fh:
context.scale_thresholds = json.load(fh)
@then("the scale test thresholds should contain a schema version")
def step_scale_test_thresholds_has_schema_version(context: Context) -> None:
assert "schema_version" in context.scale_thresholds
@then("the scale test thresholds should have indexing section")
def step_scale_test_thresholds_has_indexing(context: Context) -> None:
assert "indexing" in context.scale_thresholds
@then("the scale test thresholds should have decomposition section")
def step_scale_test_thresholds_has_decomposition(context: Context) -> None:
assert "decomposition" in context.scale_thresholds
@then('the scale test indexing thresholds should cover "{file_count}"')
def step_scale_test_indexing_covers_file_count(
context: Context, file_count: str
) -> None:
assert file_count in context.scale_thresholds["indexing"], (
f"Missing indexing threshold for {file_count}"
)
@then("each scale test indexing threshold should have p50 p95 and p99")
def step_scale_test_indexing_has_percentiles(context: Context) -> None:
for key, thresholds in context.scale_thresholds["indexing"].items():
for pct in ("p50_ms", "p95_ms", "p99_ms"):
assert pct in thresholds, f"Missing {pct} in indexing[{key}]"
assert isinstance(thresholds[pct], (int, float))
@then('the scale test decomposition thresholds should cover "{file_count}"')
def step_scale_test_decomposition_covers_file_count(
context: Context, file_count: str
) -> None:
assert file_count in context.scale_thresholds["decomposition"], (
f"Missing decomposition threshold for {file_count}"
)
@then("each scale test decomposition threshold should have p50 p95 and p99")
def step_scale_test_decomposition_has_percentiles(context: Context) -> None:
for key, thresholds in context.scale_thresholds["decomposition"].items():
for pct in ("p50_ms", "p95_ms", "p99_ms"):
assert pct in thresholds, f"Missing {pct} in decomposition[{key}]"
assert isinstance(thresholds[pct], (int, float))
@then("scale test indexing p50 should be less than p95 for all file counts")
def step_scale_test_indexing_p50_lt_p95(context: Context) -> None:
for key, thresholds in context.scale_thresholds["indexing"].items():
assert thresholds["p50_ms"] < thresholds["p95_ms"], (
f"Indexing p50 >= p95 for {key}: {thresholds['p50_ms']} >= {thresholds['p95_ms']}"
)
@then("scale test indexing p95 should be less than p99 for all file counts")
def step_scale_test_indexing_p95_lt_p99(context: Context) -> None:
for key, thresholds in context.scale_thresholds["indexing"].items():
assert thresholds["p95_ms"] < thresholds["p99_ms"], (
f"Indexing p95 >= p99 for {key}: {thresholds['p95_ms']} >= {thresholds['p99_ms']}"
)
# ---------------------------------------------------------------------------
# Section 3: Language mix distribution validation
# ---------------------------------------------------------------------------
@then("the scale test language mix should sum to 1.0 for each profile")
def step_scale_test_language_mix_sums_to_one(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
total = sum(profile["language_mix"].values())
assert math.isclose(total, 1.0, rel_tol=1e-6), (
f"Language mix for {profile['name']} sums to {total}, expected 1.0"
)
@then("each scale test language mix value should be between 0 and 1")
def step_scale_test_language_mix_values_in_range(context: Context) -> None:
for profile in context.scale_metadata["scale_profiles"]:
for lang, ratio in profile["language_mix"].items():
assert 0.0 <= ratio <= 1.0, (
f"Language mix value for {lang} in {profile['name']} is {ratio}"
)
@then("all scale test language mix keys should be in supported languages")
def step_scale_test_language_mix_keys_in_supported(context: Context) -> None:
supported = set(context.scale_metadata["supported_languages"])
for profile in context.scale_metadata["scale_profiles"]:
for lang in profile["language_mix"]:
assert lang in supported, (
f"Language {lang} in {profile['name']} mix not in supported languages"
)
# ---------------------------------------------------------------------------
# Section 4: Scale profile simulation
# ---------------------------------------------------------------------------
@when('I scale test generate file distribution for the "{profile_name}" profile')
def step_scale_test_generate_distribution(context: Context, profile_name: str) -> None:
"""Generate a file count distribution from a scale profile."""
# Load metadata if not already loaded
if context.scale_metadata is None:
path = _FIXTURES_DIR / "scale_metadata.json"
with open(path) as fh:
context.scale_metadata = json.load(fh)
profile = None
for p in context.scale_metadata["scale_profiles"]:
if p["name"] == profile_name:
profile = p
break
assert profile is not None, f"Profile {profile_name} not found"
total = profile["file_count"]
mix = profile["language_mix"]
# Distribute files according to language mix ratios
distribution: dict[str, int] = {}
allocated = 0
items = sorted(mix.items(), key=lambda x: x[1], reverse=True)
for i, (lang, ratio) in enumerate(items):
if i == len(items) - 1:
# Last language gets the remainder to ensure exact total
distribution[lang] = total - allocated
else:
count = round(total * ratio)
distribution[lang] = count
allocated += count
context.scale_file_distribution = distribution
context.scale_current_profile = profile
@then("the scale test total generated file count should be {count:d}")
def step_scale_test_total_file_count(context: Context, count: int) -> None:
total = sum(context.scale_file_distribution.values())
assert total == count, f"Expected {count} files, got {total}"
@then("the scale test generated distribution should match the language mix")
def step_scale_test_distribution_matches_mix(context: Context) -> None:
profile = context.scale_current_profile
total = profile["file_count"]
for lang, expected_ratio in profile["language_mix"].items():
actual_count = context.scale_file_distribution.get(lang, 0)
actual_ratio = actual_count / total
# Allow 5% tolerance due to rounding
assert abs(actual_ratio - expected_ratio) < 0.05, (
f"Language {lang}: expected ratio ~{expected_ratio}, "
f"got {actual_ratio} ({actual_count}/{total})"
)
# ---------------------------------------------------------------------------
# Section 5: Baseline threshold realism checks
# ---------------------------------------------------------------------------
@then("scale test 10K indexing p50 should be less than 10x the 1K p50")
def step_scale_test_sublinear_scaling(context: Context) -> None:
idx = context.scale_thresholds["indexing"]
p50_1k = idx["1000_files"]["p50_ms"]
p50_10k = idx["10000_files"]["p50_ms"]
assert p50_10k < 10 * p50_1k, (
f"10K p50 ({p50_10k}) should be < 10x 1K p50 ({10 * p50_1k})"
)
@then(
"scale test decomposition p50 should be greater than indexing p50 for each file count"
)
def step_scale_test_decomposition_gt_indexing(context: Context) -> None:
idx = context.scale_thresholds["indexing"]
dec = context.scale_thresholds["decomposition"]
for key in idx:
if key in dec:
assert dec[key]["p50_ms"] > idx[key]["p50_ms"], (
f"Decomposition p50 ({dec[key]['p50_ms']}) should exceed "
f"indexing p50 ({idx[key]['p50_ms']}) for {key}"
)
@then("the scale test thresholds should have memory usage section")
def step_scale_test_thresholds_has_memory(context: Context) -> None:
assert "memory_usage_mb" in context.scale_thresholds
@then("scale test memory peak should exceed steady state for each file count")
def step_scale_test_memory_peak_gt_steady(context: Context) -> None:
mem = context.scale_thresholds["memory_usage_mb"]
for key, values in mem.items():
assert values["peak_mb"] > values["steady_state_mb"], (
f"Peak memory ({values['peak_mb']}) should exceed "
f"steady state ({values['steady_state_mb']}) for {key}"
)
implementation_plan.md
+8 -8
View File
@@ -5055,14 +5055,14 @@ By Day 30, the system must be able to (all in LOCAL MODE, server connectivity is
- [ ] Git [Brent]: `git pull origin master`
- [ ] Git [Brent]: `git checkout -b feature/m6-perf-scale`
- [ ] Git [Brent]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
- [ ] Code [Brent]: Add scale fixtures for 1K/5K/10K file repos in `features/fixtures/scale/`.
- [ ] Code [Brent]: Add scriptless fixture generator instructions (documented, no helper scripts).
- [ ] Code [Brent]: Add baseline thresholds for indexing and decomposition runtime in a documented matrix.
- [ ] Code [Brent]: Add fixture metadata (file count, total size, language mix) for repeatable benchmarks.
- [ ] Docs [Brent]: Add scale test runbook and environment notes.
- [ ] Tests (Behave) [Brent]: Add scale test scenarios validating thresholds.
- [ ] Tests (Robot) [Brent]: Add large-project Robot tests for performance runs.
- [ ] Tests (ASV) [Brent]: Add `benchmarks/scale_fixture_bench.py` for baseline performance.
- [x] Code [Brent]: Add scale fixtures for 1K/5K/10K file repos in `features/fixtures/scale/`.
- [x] Code [Brent]: Add scriptless fixture generator instructions (documented, no helper scripts).
- [x] Code [Brent]: Add baseline thresholds for indexing and decomposition runtime in a documented matrix.
- [x] Code [Brent]: Add fixture metadata (file count, total size, language mix) for repeatable benchmarks.
- [x] Docs [Brent]: Add scale test runbook and environment notes.
- [x] Tests (Behave) [Brent]: Add scale test scenarios validating thresholds.
- [x] Tests (Robot) [Brent]: Add large-project Robot tests for performance runs.
- [x] Tests (ASV) [Brent]: Add `benchmarks/scale_fixture_bench.py` for baseline performance.
- [ ] Quality [Brent]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
- [ ] Quality [Brent]: Run `nox` (all default sessions, including benchmark).
- [ ] Git [Brent]: `git add .` (only after nox passes)
robot/helper_scale_test.py
+193
View File
@@ -0,0 +1,193 @@
"""Robot Framework helper for scale test fixture validation.
Provides a CLI-style interface for Robot to invoke fixture loading,
metadata validation, threshold checks, and profile simulation.
Exit code 0 = success, 1 = failure.
Usage:
python robot/helper_scale_test.py load-metadata
python robot/helper_scale_test.py validate-profiles
python robot/helper_scale_test.py validate-thresholds
python robot/helper_scale_test.py check-monotonicity
python robot/helper_scale_test.py simulate-distribution
python robot/helper_scale_test.py check-generator-docs
"""
from __future__ import annotations
import json
import math
import sys
from pathlib import Path
# Ensure the src directory is on the import path.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "scale"
def _load_json(filename: str) -> dict:
"""Load a JSON fixture file."""
path = _FIXTURES_DIR / filename
if not path.exists():
print(f"ERROR: fixture not found: {path}")
sys.exit(1)
with open(path) as fh:
return json.load(fh)
def _cmd_load_metadata() -> int:
"""Load scale_metadata.json and verify basic structure."""
data = _load_json("scale_metadata.json")
assert "schema_version" in data, "Missing schema_version"
assert "scale_profiles" in data, "Missing scale_profiles"
assert len(data["scale_profiles"]) > 0, "No profiles defined"
print(f"scale-metadata-ok: {len(data['scale_profiles'])} profiles loaded")
return 0
def _cmd_validate_profiles() -> int:
"""Validate all scale profiles have required fields."""
data = _load_json("scale_metadata.json")
required_fields = [
"name",
"file_count",
"total_size_mb",
"language_mix",
"expected_index_time_s",
"expected_decomposition_time_s",
]
expected_names = {"small", "medium", "large"}
actual_names = {p["name"] for p in data["scale_profiles"]}
assert actual_names == expected_names, (
f"Expected profiles {expected_names}, got {actual_names}"
)
for profile in data["scale_profiles"]:
for field in required_fields:
assert field in profile, f"Missing {field} in profile {profile.get('name')}"
# Verify language mix sums to 1.0
mix_total = sum(profile["language_mix"].values())
assert math.isclose(mix_total, 1.0, rel_tol=1e-6), (
f"Language mix for {profile['name']} sums to {mix_total}"
)
print(f"scale-profiles-ok: {len(data['scale_profiles'])} profiles validated")
return 0
def _cmd_validate_thresholds() -> int:
"""Load and validate baseline thresholds structure."""
data = _load_json("baseline_thresholds.json")
assert "schema_version" in data, "Missing schema_version"
for section in ("indexing", "decomposition"):
assert section in data, f"Missing section: {section}"
for file_count in ("1000_files", "5000_files", "10000_files"):
assert file_count in data[section], f"Missing {file_count} in {section}"
for pct in ("p50_ms", "p95_ms", "p99_ms"):
assert pct in data[section][file_count], (
f"Missing {pct} in {section}[{file_count}]"
)
print("scale-thresholds-ok: all threshold sections validated")
return 0
def _cmd_check_monotonicity() -> int:
"""Verify p50 < p95 < p99 for all threshold entries."""
data = _load_json("baseline_thresholds.json")
for section in ("indexing", "decomposition"):
for file_count, thresholds in data[section].items():
p50 = thresholds["p50_ms"]
p95 = thresholds["p95_ms"]
p99 = thresholds["p99_ms"]
assert p50 < p95 < p99, (
f"Non-monotonic in {section}[{file_count}]: "
f"p50={p50}, p95={p95}, p99={p99}"
)
print("scale-monotonicity-ok: all thresholds are monotonically increasing")
return 0
def _cmd_simulate_distribution() -> int:
"""Generate file distributions for all profiles and verify totals."""
data = _load_json("scale_metadata.json")
for profile in data["scale_profiles"]:
total = profile["file_count"]
mix = profile["language_mix"]
distribution: dict[str, int] = {}
allocated = 0
items = sorted(mix.items(), key=lambda x: x[1], reverse=True)
for i, (lang, ratio) in enumerate(items):
if i == len(items) - 1:
distribution[lang] = total - allocated
else:
count = round(total * ratio)
distribution[lang] = count
allocated += count
actual_total = sum(distribution.values())
assert actual_total == total, (
f"Profile {profile['name']}: expected {total} files, got {actual_total}"
)
# Verify distribution roughly matches mix
for lang, expected_ratio in mix.items():
actual_ratio = distribution[lang] / total
assert abs(actual_ratio - expected_ratio) < 0.05, (
f"Profile {profile['name']}, lang {lang}: "
f"expected ~{expected_ratio}, got {actual_ratio}"
)
print(f" {profile['name']}: {distribution}")
print("scale-distribution-ok: all profile distributions validated")
return 0
def _cmd_check_generator_docs() -> int:
"""Verify the generator instructions document exists and has content."""
path = _FIXTURES_DIR / "generator_instructions.md"
assert path.exists(), f"Generator instructions not found: {path}"
content = path.read_text()
assert len(content) > 100, "Generator instructions file is too short"
assert "## Overview" in content, "Missing Overview section"
assert "## Generating" in content or "## Prerequisites" in content, (
"Missing generation instructions"
)
print("scale-generator-docs-ok: generator instructions validated")
return 0
def main() -> int:
"""Entry point called by Robot Framework ``Run Process``."""
if len(sys.argv) < 2:
print(
"Usage: helper_scale_test.py "
"<load-metadata|validate-profiles|validate-thresholds|"
"check-monotonicity|simulate-distribution|check-generator-docs>"
)
return 1
command = sys.argv[1]
commands = {
"load-metadata": _cmd_load_metadata,
"validate-profiles": _cmd_validate_profiles,
"validate-thresholds": _cmd_validate_thresholds,
"check-monotonicity": _cmd_check_monotonicity,
"simulate-distribution": _cmd_simulate_distribution,
"check-generator-docs": _cmd_check_generator_docs,
}
handler = commands.get(command)
if handler is None:
print(f"Unknown command: {command}")
return 1
try:
return handler()
except (AssertionError, KeyError, json.JSONDecodeError) as exc:
print(f"FAIL: {exc}")
return 1
if __name__ == "__main__":
sys.exit(main())
robot/scale_test.robot
+59
View File
@@ -0,0 +1,59 @@
*** Settings ***
Documentation Scale test fixture validation and performance baseline checks.
... Validates fixture loading, metadata structure, threshold matrices,
... and profile generation simulation.
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Variables ***
${HELPER} ${CURDIR}/helper_scale_test.py
*** Test Cases ***
Load Scale Metadata Fixture
[Documentation] Load scale_metadata.json and verify structure
${result}= Run Process ${PYTHON} ${HELPER} load-metadata cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-metadata-ok
Validate Scale Metadata Profiles
[Documentation] Verify all expected scale profiles are present with correct fields
${result}= Run Process ${PYTHON} ${HELPER} validate-profiles cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-profiles-ok
Load And Validate Baseline Thresholds
[Documentation] Load baseline_thresholds.json and verify threshold matrix structure
${result}= Run Process ${PYTHON} ${HELPER} validate-thresholds cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-thresholds-ok
Verify Threshold Monotonicity
[Documentation] Verify p50 < p95 < p99 across all threshold entries
${result}= Run Process ${PYTHON} ${HELPER} check-monotonicity cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-monotonicity-ok
Simulate Profile File Distribution
[Documentation] Generate file count distributions and verify they match language mix
${result}= Run Process ${PYTHON} ${HELPER} simulate-distribution cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-distribution-ok
Validate Generator Instructions Exist
[Documentation] Verify the generator instructions document is present
${result}= Run Process ${PYTHON} ${HELPER} check-generator-docs cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} scale-generator-docs-ok
src/cleveragents/application/services/audit_service.py
+4 -1
View File
@@ -207,7 +207,10 @@ class AuditService:
if since is not None:
query = query.filter(AuditLogModel.created_at >= since)
query = query.order_by(AuditLogModel.created_at.desc()).limit(limit)
query = query.order_by(
AuditLogModel.created_at.desc(),
AuditLogModel.id.desc(),
).limit(limit)
return [self._row_to_entry(row) for row in query.all()]