"""Step definitions for scale test fixture validation.""" from __future__ import annotations import json import math from pathlib import Path from behave import given, then, when from behave.runner import Context _FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "scale" # --------------------------------------------------------------------------- # Background # --------------------------------------------------------------------------- @given("a scale test environment is ready") def step_scale_test_environment_ready(context: Context) -> None: """Verify scale test fixtures directory exists.""" assert _FIXTURES_DIR.is_dir(), f"Fixtures dir missing: {_FIXTURES_DIR}" context.scale_metadata = None context.scale_thresholds = None context.scale_file_distribution = None # --------------------------------------------------------------------------- # Section 1: Fixture loading and metadata validation # --------------------------------------------------------------------------- @when("I scale test load the scale metadata fixture") def step_scale_test_load_metadata(context: Context) -> None: """Load scale_metadata.json from fixtures.""" path = _FIXTURES_DIR / "scale_metadata.json" assert path.exists(), f"Missing fixture: {path}" with open(path) as fh: context.scale_metadata = json.load(fh) @then("the scale test metadata should contain a schema version") def step_scale_test_metadata_has_schema_version(context: Context) -> None: assert "schema_version" in context.scale_metadata @then("the scale test metadata should contain scale profiles") def step_scale_test_metadata_has_profiles(context: Context) -> None: assert "scale_profiles" in context.scale_metadata assert len(context.scale_metadata["scale_profiles"]) > 0 @then("the scale test metadata should have exactly {count:d} profiles") def step_scale_test_metadata_profile_count(context: Context, count: int) -> None: assert len(context.scale_metadata["scale_profiles"]) == count @then('the scale test profile names should be "{names}"') def step_scale_test_profile_names(context: Context, names: str) -> None: expected = [n.strip() for n in names.split(",")] actual = [p["name"] for p in context.scale_metadata["scale_profiles"]] assert actual == expected, f"Expected {expected}, got {actual}" @then("each scale test profile should have a file count") def step_scale_test_profile_has_file_count(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: assert "file_count" in profile, f"Missing file_count in {profile['name']}" assert isinstance(profile["file_count"], int) @then("each scale test profile should have a total size in megabytes") def step_scale_test_profile_has_total_size(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: assert "total_size_mb" in profile, f"Missing total_size_mb in {profile['name']}" assert isinstance(profile["total_size_mb"], (int, float)) @then("each scale test profile should have a language mix") def step_scale_test_profile_has_language_mix(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: assert "language_mix" in profile, f"Missing language_mix in {profile['name']}" assert isinstance(profile["language_mix"], dict) @then("each scale test profile should have expected index time") def step_scale_test_profile_has_index_time(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: assert "expected_index_time_s" in profile, ( f"Missing expected_index_time_s in {profile['name']}" ) @then("each scale test profile should have expected decomposition time") def step_scale_test_profile_has_decomposition_time(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: assert "expected_decomposition_time_s" in profile, ( f"Missing expected_decomposition_time_s in {profile['name']}" ) @then("the scale test metadata should list supported languages") def step_scale_test_metadata_has_supported_languages(context: Context) -> None: assert "supported_languages" in context.scale_metadata assert len(context.scale_metadata["supported_languages"]) > 0 @then('the scale test supported languages should include "{language}"') def step_scale_test_supported_languages_include( context: Context, language: str ) -> None: assert language in context.scale_metadata["supported_languages"] # --------------------------------------------------------------------------- # Section 2: Threshold matrix validation # --------------------------------------------------------------------------- @when("I scale test load the baseline thresholds fixture") def step_scale_test_load_thresholds(context: Context) -> None: """Load baseline_thresholds.json from fixtures.""" path = _FIXTURES_DIR / "baseline_thresholds.json" assert path.exists(), f"Missing fixture: {path}" with open(path) as fh: context.scale_thresholds = json.load(fh) @then("the scale test thresholds should contain a schema version") def step_scale_test_thresholds_has_schema_version(context: Context) -> None: assert "schema_version" in context.scale_thresholds @then("the scale test thresholds should have indexing section") def step_scale_test_thresholds_has_indexing(context: Context) -> None: assert "indexing" in context.scale_thresholds @then("the scale test thresholds should have decomposition section") def step_scale_test_thresholds_has_decomposition(context: Context) -> None: assert "decomposition" in context.scale_thresholds @then('the scale test indexing thresholds should cover "{file_count}"') def step_scale_test_indexing_covers_file_count( context: Context, file_count: str ) -> None: assert file_count in context.scale_thresholds["indexing"], ( f"Missing indexing threshold for {file_count}" ) @then("each scale test indexing threshold should have p50 p95 and p99") def step_scale_test_indexing_has_percentiles(context: Context) -> None: for key, thresholds in context.scale_thresholds["indexing"].items(): for pct in ("p50_ms", "p95_ms", "p99_ms"): assert pct in thresholds, f"Missing {pct} in indexing[{key}]" assert isinstance(thresholds[pct], (int, float)) @then('the scale test decomposition thresholds should cover "{file_count}"') def step_scale_test_decomposition_covers_file_count( context: Context, file_count: str ) -> None: assert file_count in context.scale_thresholds["decomposition"], ( f"Missing decomposition threshold for {file_count}" ) @then("each scale test decomposition threshold should have p50 p95 and p99") def step_scale_test_decomposition_has_percentiles(context: Context) -> None: for key, thresholds in context.scale_thresholds["decomposition"].items(): for pct in ("p50_ms", "p95_ms", "p99_ms"): assert pct in thresholds, f"Missing {pct} in decomposition[{key}]" assert isinstance(thresholds[pct], (int, float)) @then("scale test indexing p50 should be less than p95 for all file counts") def step_scale_test_indexing_p50_lt_p95(context: Context) -> None: for key, thresholds in context.scale_thresholds["indexing"].items(): assert thresholds["p50_ms"] < thresholds["p95_ms"], ( f"Indexing p50 >= p95 for {key}: {thresholds['p50_ms']} >= {thresholds['p95_ms']}" ) @then("scale test indexing p95 should be less than p99 for all file counts") def step_scale_test_indexing_p95_lt_p99(context: Context) -> None: for key, thresholds in context.scale_thresholds["indexing"].items(): assert thresholds["p95_ms"] < thresholds["p99_ms"], ( f"Indexing p95 >= p99 for {key}: {thresholds['p95_ms']} >= {thresholds['p99_ms']}" ) # --------------------------------------------------------------------------- # Section 3: Language mix distribution validation # --------------------------------------------------------------------------- @then("the scale test language mix should sum to 1.0 for each profile") def step_scale_test_language_mix_sums_to_one(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: total = sum(profile["language_mix"].values()) assert math.isclose(total, 1.0, rel_tol=1e-6), ( f"Language mix for {profile['name']} sums to {total}, expected 1.0" ) @then("each scale test language mix value should be between 0 and 1") def step_scale_test_language_mix_values_in_range(context: Context) -> None: for profile in context.scale_metadata["scale_profiles"]: for lang, ratio in profile["language_mix"].items(): assert 0.0 <= ratio <= 1.0, ( f"Language mix value for {lang} in {profile['name']} is {ratio}" ) @then("all scale test language mix keys should be in supported languages") def step_scale_test_language_mix_keys_in_supported(context: Context) -> None: supported = set(context.scale_metadata["supported_languages"]) for profile in context.scale_metadata["scale_profiles"]: for lang in profile["language_mix"]: assert lang in supported, ( f"Language {lang} in {profile['name']} mix not in supported languages" ) # --------------------------------------------------------------------------- # Section 4: Scale profile simulation # --------------------------------------------------------------------------- @when('I scale test generate file distribution for the "{profile_name}" profile') def step_scale_test_generate_distribution(context: Context, profile_name: str) -> None: """Generate a file count distribution from a scale profile.""" # Load metadata if not already loaded if context.scale_metadata is None: path = _FIXTURES_DIR / "scale_metadata.json" with open(path) as fh: context.scale_metadata = json.load(fh) profile = None for p in context.scale_metadata["scale_profiles"]: if p["name"] == profile_name: profile = p break assert profile is not None, f"Profile {profile_name} not found" total = profile["file_count"] mix = profile["language_mix"] # Distribute files according to language mix ratios distribution: dict[str, int] = {} allocated = 0 items = sorted(mix.items(), key=lambda x: x[1], reverse=True) for i, (lang, ratio) in enumerate(items): if i == len(items) - 1: # Last language gets the remainder to ensure exact total distribution[lang] = total - allocated else: count = round(total * ratio) distribution[lang] = count allocated += count context.scale_file_distribution = distribution context.scale_current_profile = profile @then("the scale test total generated file count should be {count:d}") def step_scale_test_total_file_count(context: Context, count: int) -> None: total = sum(context.scale_file_distribution.values()) assert total == count, f"Expected {count} files, got {total}" @then("the scale test generated distribution should match the language mix") def step_scale_test_distribution_matches_mix(context: Context) -> None: profile = context.scale_current_profile total = profile["file_count"] for lang, expected_ratio in profile["language_mix"].items(): actual_count = context.scale_file_distribution.get(lang, 0) actual_ratio = actual_count / total # Allow 5% tolerance due to rounding assert abs(actual_ratio - expected_ratio) < 0.05, ( f"Language {lang}: expected ratio ~{expected_ratio}, " f"got {actual_ratio} ({actual_count}/{total})" ) # --------------------------------------------------------------------------- # Section 5: Baseline threshold realism checks # --------------------------------------------------------------------------- @then("scale test 10K indexing p50 should be less than 10x the 1K p50") def step_scale_test_sublinear_scaling(context: Context) -> None: idx = context.scale_thresholds["indexing"] p50_1k = idx["1000_files"]["p50_ms"] p50_10k = idx["10000_files"]["p50_ms"] assert p50_10k < 10 * p50_1k, ( f"10K p50 ({p50_10k}) should be < 10x 1K p50 ({10 * p50_1k})" ) @then( "scale test decomposition p50 should be greater than indexing p50 for each file count" ) def step_scale_test_decomposition_gt_indexing(context: Context) -> None: idx = context.scale_thresholds["indexing"] dec = context.scale_thresholds["decomposition"] for key in idx: if key in dec: assert dec[key]["p50_ms"] > idx[key]["p50_ms"], ( f"Decomposition p50 ({dec[key]['p50_ms']}) should exceed " f"indexing p50 ({idx[key]['p50_ms']}) for {key}" ) @then("the scale test thresholds should have memory usage section") def step_scale_test_thresholds_has_memory(context: Context) -> None: assert "memory_usage_mb" in context.scale_thresholds @then("scale test memory peak should exceed steady state for each file count") def step_scale_test_memory_peak_gt_steady(context: Context) -> None: mem = context.scale_thresholds["memory_usage_mb"] for key, values in mem.items(): assert values["peak_mb"] > values["steady_state_mb"], ( f"Peak memory ({values['peak_mb']}) should exceed " f"steady state ({values['steady_state_mb']}) for {key}" )