331 lines
13 KiB
Python
331 lines
13 KiB
Python
"""Step definitions for scale test fixture validation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "scale"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a scale test environment is ready")
|
|
def step_scale_test_environment_ready(context: Context) -> None:
|
|
"""Verify scale test fixtures directory exists."""
|
|
assert _FIXTURES_DIR.is_dir(), f"Fixtures dir missing: {_FIXTURES_DIR}"
|
|
context.scale_metadata = None
|
|
context.scale_thresholds = None
|
|
context.scale_file_distribution = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 1: Fixture loading and metadata validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I scale test load the scale metadata fixture")
|
|
def step_scale_test_load_metadata(context: Context) -> None:
|
|
"""Load scale_metadata.json from fixtures."""
|
|
path = _FIXTURES_DIR / "scale_metadata.json"
|
|
assert path.exists(), f"Missing fixture: {path}"
|
|
with open(path) as fh:
|
|
context.scale_metadata = json.load(fh)
|
|
|
|
|
|
@then("the scale test metadata should contain a schema version")
|
|
def step_scale_test_metadata_has_schema_version(context: Context) -> None:
|
|
assert "schema_version" in context.scale_metadata
|
|
|
|
|
|
@then("the scale test metadata should contain scale profiles")
|
|
def step_scale_test_metadata_has_profiles(context: Context) -> None:
|
|
assert "scale_profiles" in context.scale_metadata
|
|
assert len(context.scale_metadata["scale_profiles"]) > 0
|
|
|
|
|
|
@then("the scale test metadata should have exactly {count:d} profiles")
|
|
def step_scale_test_metadata_profile_count(context: Context, count: int) -> None:
|
|
assert len(context.scale_metadata["scale_profiles"]) == count
|
|
|
|
|
|
@then('the scale test profile names should be "{names}"')
|
|
def step_scale_test_profile_names(context: Context, names: str) -> None:
|
|
expected = [n.strip() for n in names.split(",")]
|
|
actual = [p["name"] for p in context.scale_metadata["scale_profiles"]]
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("each scale test profile should have a file count")
|
|
def step_scale_test_profile_has_file_count(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
assert "file_count" in profile, f"Missing file_count in {profile['name']}"
|
|
assert isinstance(profile["file_count"], int)
|
|
|
|
|
|
@then("each scale test profile should have a total size in megabytes")
|
|
def step_scale_test_profile_has_total_size(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
assert "total_size_mb" in profile, f"Missing total_size_mb in {profile['name']}"
|
|
assert isinstance(profile["total_size_mb"], (int, float))
|
|
|
|
|
|
@then("each scale test profile should have a language mix")
|
|
def step_scale_test_profile_has_language_mix(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
assert "language_mix" in profile, f"Missing language_mix in {profile['name']}"
|
|
assert isinstance(profile["language_mix"], dict)
|
|
|
|
|
|
@then("each scale test profile should have expected index time")
|
|
def step_scale_test_profile_has_index_time(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
assert "expected_index_time_s" in profile, (
|
|
f"Missing expected_index_time_s in {profile['name']}"
|
|
)
|
|
|
|
|
|
@then("each scale test profile should have expected decomposition time")
|
|
def step_scale_test_profile_has_decomposition_time(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
assert "expected_decomposition_time_s" in profile, (
|
|
f"Missing expected_decomposition_time_s in {profile['name']}"
|
|
)
|
|
|
|
|
|
@then("the scale test metadata should list supported languages")
|
|
def step_scale_test_metadata_has_supported_languages(context: Context) -> None:
|
|
assert "supported_languages" in context.scale_metadata
|
|
assert len(context.scale_metadata["supported_languages"]) > 0
|
|
|
|
|
|
@then('the scale test supported languages should include "{language}"')
|
|
def step_scale_test_supported_languages_include(
|
|
context: Context, language: str
|
|
) -> None:
|
|
assert language in context.scale_metadata["supported_languages"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 2: Threshold matrix validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I scale test load the baseline thresholds fixture")
|
|
def step_scale_test_load_thresholds(context: Context) -> None:
|
|
"""Load baseline_thresholds.json from fixtures."""
|
|
path = _FIXTURES_DIR / "baseline_thresholds.json"
|
|
assert path.exists(), f"Missing fixture: {path}"
|
|
with open(path) as fh:
|
|
context.scale_thresholds = json.load(fh)
|
|
|
|
|
|
@then("the scale test thresholds should contain a schema version")
|
|
def step_scale_test_thresholds_has_schema_version(context: Context) -> None:
|
|
assert "schema_version" in context.scale_thresholds
|
|
|
|
|
|
@then("the scale test thresholds should have indexing section")
|
|
def step_scale_test_thresholds_has_indexing(context: Context) -> None:
|
|
assert "indexing" in context.scale_thresholds
|
|
|
|
|
|
@then("the scale test thresholds should have decomposition section")
|
|
def step_scale_test_thresholds_has_decomposition(context: Context) -> None:
|
|
assert "decomposition" in context.scale_thresholds
|
|
|
|
|
|
@then('the scale test indexing thresholds should cover "{file_count}"')
|
|
def step_scale_test_indexing_covers_file_count(
|
|
context: Context, file_count: str
|
|
) -> None:
|
|
assert file_count in context.scale_thresholds["indexing"], (
|
|
f"Missing indexing threshold for {file_count}"
|
|
)
|
|
|
|
|
|
@then("each scale test indexing threshold should have p50 p95 and p99")
|
|
def step_scale_test_indexing_has_percentiles(context: Context) -> None:
|
|
for key, thresholds in context.scale_thresholds["indexing"].items():
|
|
for pct in ("p50_ms", "p95_ms", "p99_ms"):
|
|
assert pct in thresholds, f"Missing {pct} in indexing[{key}]"
|
|
assert isinstance(thresholds[pct], (int, float))
|
|
|
|
|
|
@then('the scale test decomposition thresholds should cover "{file_count}"')
|
|
def step_scale_test_decomposition_covers_file_count(
|
|
context: Context, file_count: str
|
|
) -> None:
|
|
assert file_count in context.scale_thresholds["decomposition"], (
|
|
f"Missing decomposition threshold for {file_count}"
|
|
)
|
|
|
|
|
|
@then("each scale test decomposition threshold should have p50 p95 and p99")
|
|
def step_scale_test_decomposition_has_percentiles(context: Context) -> None:
|
|
for key, thresholds in context.scale_thresholds["decomposition"].items():
|
|
for pct in ("p50_ms", "p95_ms", "p99_ms"):
|
|
assert pct in thresholds, f"Missing {pct} in decomposition[{key}]"
|
|
assert isinstance(thresholds[pct], (int, float))
|
|
|
|
|
|
@then("scale test indexing p50 should be less than p95 for all file counts")
|
|
def step_scale_test_indexing_p50_lt_p95(context: Context) -> None:
|
|
for key, thresholds in context.scale_thresholds["indexing"].items():
|
|
assert thresholds["p50_ms"] < thresholds["p95_ms"], (
|
|
f"Indexing p50 >= p95 for {key}: {thresholds['p50_ms']} >= {thresholds['p95_ms']}"
|
|
)
|
|
|
|
|
|
@then("scale test indexing p95 should be less than p99 for all file counts")
|
|
def step_scale_test_indexing_p95_lt_p99(context: Context) -> None:
|
|
for key, thresholds in context.scale_thresholds["indexing"].items():
|
|
assert thresholds["p95_ms"] < thresholds["p99_ms"], (
|
|
f"Indexing p95 >= p99 for {key}: {thresholds['p95_ms']} >= {thresholds['p99_ms']}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 3: Language mix distribution validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the scale test language mix should sum to 1.0 for each profile")
|
|
def step_scale_test_language_mix_sums_to_one(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
total = sum(profile["language_mix"].values())
|
|
assert math.isclose(total, 1.0, rel_tol=1e-6), (
|
|
f"Language mix for {profile['name']} sums to {total}, expected 1.0"
|
|
)
|
|
|
|
|
|
@then("each scale test language mix value should be between 0 and 1")
|
|
def step_scale_test_language_mix_values_in_range(context: Context) -> None:
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
for lang, ratio in profile["language_mix"].items():
|
|
assert 0.0 <= ratio <= 1.0, (
|
|
f"Language mix value for {lang} in {profile['name']} is {ratio}"
|
|
)
|
|
|
|
|
|
@then("all scale test language mix keys should be in supported languages")
|
|
def step_scale_test_language_mix_keys_in_supported(context: Context) -> None:
|
|
supported = set(context.scale_metadata["supported_languages"])
|
|
for profile in context.scale_metadata["scale_profiles"]:
|
|
for lang in profile["language_mix"]:
|
|
assert lang in supported, (
|
|
f"Language {lang} in {profile['name']} mix not in supported languages"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 4: Scale profile simulation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I scale test generate file distribution for the "{profile_name}" profile')
|
|
def step_scale_test_generate_distribution(context: Context, profile_name: str) -> None:
|
|
"""Generate a file count distribution from a scale profile."""
|
|
# Load metadata if not already loaded
|
|
if context.scale_metadata is None:
|
|
path = _FIXTURES_DIR / "scale_metadata.json"
|
|
with open(path) as fh:
|
|
context.scale_metadata = json.load(fh)
|
|
|
|
profile = None
|
|
for p in context.scale_metadata["scale_profiles"]:
|
|
if p["name"] == profile_name:
|
|
profile = p
|
|
break
|
|
assert profile is not None, f"Profile {profile_name} not found"
|
|
|
|
total = profile["file_count"]
|
|
mix = profile["language_mix"]
|
|
|
|
# Distribute files according to language mix ratios
|
|
distribution: dict[str, int] = {}
|
|
allocated = 0
|
|
items = sorted(mix.items(), key=lambda x: x[1], reverse=True)
|
|
for i, (lang, ratio) in enumerate(items):
|
|
if i == len(items) - 1:
|
|
# Last language gets the remainder to ensure exact total
|
|
distribution[lang] = total - allocated
|
|
else:
|
|
count = round(total * ratio)
|
|
distribution[lang] = count
|
|
allocated += count
|
|
|
|
context.scale_file_distribution = distribution
|
|
context.scale_current_profile = profile
|
|
|
|
|
|
@then("the scale test total generated file count should be {count:d}")
|
|
def step_scale_test_total_file_count(context: Context, count: int) -> None:
|
|
total = sum(context.scale_file_distribution.values())
|
|
assert total == count, f"Expected {count} files, got {total}"
|
|
|
|
|
|
@then("the scale test generated distribution should match the language mix")
|
|
def step_scale_test_distribution_matches_mix(context: Context) -> None:
|
|
profile = context.scale_current_profile
|
|
total = profile["file_count"]
|
|
for lang, expected_ratio in profile["language_mix"].items():
|
|
actual_count = context.scale_file_distribution.get(lang, 0)
|
|
actual_ratio = actual_count / total
|
|
# Allow 5% tolerance due to rounding
|
|
assert abs(actual_ratio - expected_ratio) < 0.05, (
|
|
f"Language {lang}: expected ratio ~{expected_ratio}, "
|
|
f"got {actual_ratio} ({actual_count}/{total})"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 5: Baseline threshold realism checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("scale test 10K indexing p50 should be less than 10x the 1K p50")
|
|
def step_scale_test_sublinear_scaling(context: Context) -> None:
|
|
idx = context.scale_thresholds["indexing"]
|
|
p50_1k = idx["1000_files"]["p50_ms"]
|
|
p50_10k = idx["10000_files"]["p50_ms"]
|
|
assert p50_10k < 10 * p50_1k, (
|
|
f"10K p50 ({p50_10k}) should be < 10x 1K p50 ({10 * p50_1k})"
|
|
)
|
|
|
|
|
|
@then(
|
|
"scale test decomposition p50 should be greater than indexing p50 for each file count"
|
|
)
|
|
def step_scale_test_decomposition_gt_indexing(context: Context) -> None:
|
|
idx = context.scale_thresholds["indexing"]
|
|
dec = context.scale_thresholds["decomposition"]
|
|
for key in idx:
|
|
if key in dec:
|
|
assert dec[key]["p50_ms"] > idx[key]["p50_ms"], (
|
|
f"Decomposition p50 ({dec[key]['p50_ms']}) should exceed "
|
|
f"indexing p50 ({idx[key]['p50_ms']}) for {key}"
|
|
)
|
|
|
|
|
|
@then("the scale test thresholds should have memory usage section")
|
|
def step_scale_test_thresholds_has_memory(context: Context) -> None:
|
|
assert "memory_usage_mb" in context.scale_thresholds
|
|
|
|
|
|
@then("scale test memory peak should exceed steady state for each file count")
|
|
def step_scale_test_memory_peak_gt_steady(context: Context) -> None:
|
|
mem = context.scale_thresholds["memory_usage_mb"]
|
|
for key, values in mem.items():
|
|
assert values["peak_mb"] > values["steady_state_mb"], (
|
|
f"Peak memory ({values['peak_mb']}) should exceed "
|
|
f"steady state ({values['steady_state_mb']}) for {key}"
|
|
)
|