01b6eb1804
CI / build (push) Successful in 17s
CI / helm (push) Successful in 22s
CI / lint (push) Successful in 28s
CI / typecheck (push) Successful in 47s
CI / benchmark-regression (push) Has been skipped
CI / quality (push) Successful in 3m49s
CI / security (push) Successful in 4m11s
CI / unit_tests (push) Successful in 9m19s
CI / docker (push) Successful in 1m22s
CI / coverage (push) Successful in 12m35s
CI / e2e_tests (push) Successful in 16m17s
CI / integration_tests (push) Successful in 25m5s
CI / status-check (push) Successful in 2s
CI / benchmark-publish (push) Successful in 28m31s
## Summary Add M6 parallel-scaling coverage for 10+ concurrent subplans: - **15-subplan parallel scenario** with explicit peak-concurrency bound checks (`max_parallel=10`) and thread-safe concurrency tracking via `_build_executor()`. - **Deep hierarchical decomposition** coverage (4+ levels) with adjusted leaf condition that only stops early when hitting `max_depth` or when the workset is trivially small (`min_files_per_subplan`). - **Non-progress guard** in `_build_hierarchy` to prevent pathological recursion when clustering cannot meaningfully split the file set. - **Small-project regression test** (< 50 files) verifying decomposition depth does not increase unexpectedly with the relaxed leaf condition. - **ASV benchmark** for 15-subplan parallel execution with `max_parallel=10` to track scaling behavior. ### Removed from this PR The `_build_hierarchy` child-linkage correctness fix (returning `node_id` from recursive calls instead of using `nodes[-1].node_id`) has been **removed** per review feedback — it is a separate bug fix and will be submitted as an independent issue/PR per CONTRIBUTING.md §Atomic Commits. ## Approach - **Concurrency tracking:** The `_build_executor()` closure in step definitions detects `context.concurrency_counter` / `context.concurrency_lock` and performs thread-safe peak tracking in a try/finally block. - **Leaf condition:** Replaced the `max_files_per_subplan` / `max_tokens_per_subplan` leaf check with a `min_files_per_subplan` check to allow deeper decomposition for large projects. Added a non-progress guard so clustering that cannot split the file set terminates immediately rather than recursing to `max_depth`. - **Deterministic IDs:** `_ids_for_count()` preserves legacy fixed IDs for the first 5 subplans and generates additional deterministic IDs for scale scenarios. ## Validation ### Passing - `nox -s lint` — all checks passed - `nox -s typecheck` — 0 errors, 0 warnings - `nox -s unit_tests` — 12,988 scenarios passed, 0 failed - `nox -s coverage_report` — 97% (passes `--fail-under=97`) Closes #855 Reviewed-on: #1201 Co-authored-by: Brent E. Edwards <brent.edwards@cleverthis.com> Co-committed-by: Brent E. Edwards <brent.edwards@cleverthis.com>
143 lines
4.1 KiB
Python
143 lines
4.1 KiB
Python
"""Helper library for CLI consistency Robot Framework tests.
|
|
|
|
Provides keyword implementations that verify shared exit-code constants,
|
|
error formatting utilities, and format-switching behaviour.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
def get_exit_code_constants() -> dict[str, int]:
|
|
"""Return the CLI exit code constants as a dict."""
|
|
from cleveragents.cli.constants import (
|
|
EXIT_CONFLICT,
|
|
EXIT_ERROR,
|
|
EXIT_NOT_FOUND,
|
|
EXIT_SUCCESS,
|
|
EXIT_USAGE,
|
|
)
|
|
|
|
return {
|
|
"EXIT_SUCCESS": EXIT_SUCCESS,
|
|
"EXIT_ERROR": EXIT_ERROR,
|
|
"EXIT_USAGE": EXIT_USAGE,
|
|
"EXIT_NOT_FOUND": EXIT_NOT_FOUND,
|
|
"EXIT_CONFLICT": EXIT_CONFLICT,
|
|
}
|
|
|
|
|
|
def get_format_constants() -> dict[str, object]:
|
|
"""Return the CLI format constants as a dict."""
|
|
from cleveragents.cli.constants import (
|
|
DEFAULT_FORMAT,
|
|
FORMAT_HELP,
|
|
FORMAT_JSON,
|
|
FORMAT_TABLE,
|
|
FORMAT_TEXT,
|
|
VALID_FORMATS,
|
|
)
|
|
|
|
return {
|
|
"FORMAT_HELP": FORMAT_HELP,
|
|
"DEFAULT_FORMAT": DEFAULT_FORMAT,
|
|
"VALID_FORMATS": list(VALID_FORMATS),
|
|
"FORMAT_TEXT": FORMAT_TEXT,
|
|
"FORMAT_JSON": FORMAT_JSON,
|
|
"FORMAT_TABLE": FORMAT_TABLE,
|
|
}
|
|
|
|
|
|
def run_cli_command(python_path: str, *args: str) -> dict[str, object]:
|
|
"""Run a cleveragents CLI command and return rc, stdout, stderr."""
|
|
cmd = [python_path, "-m", "cleveragents", *args]
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
return {
|
|
"rc": result.returncode,
|
|
"stdout": result.stdout,
|
|
"stderr": result.stderr,
|
|
}
|
|
|
|
|
|
def is_valid_json(text: str) -> bool:
|
|
"""Return True if *text* is valid JSON."""
|
|
try:
|
|
json.loads(text)
|
|
except (json.JSONDecodeError, TypeError, ValueError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _run_error_script(python_path: str, script: str) -> dict[str, object]:
|
|
"""Execute a multi-line Python script in a subprocess."""
|
|
result = subprocess.run(
|
|
[python_path, "-c", script],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=90,
|
|
)
|
|
return {
|
|
"rc": result.returncode,
|
|
"stdout": result.stdout,
|
|
"stderr": result.stderr,
|
|
}
|
|
|
|
|
|
def verify_cli_error_format(python_path: str) -> dict[str, object]:
|
|
"""Run a helper script that exercises cli_error and capture output."""
|
|
target = "cleveragents.cli.errors._get_err_console"
|
|
script = "\n".join(
|
|
[
|
|
"import sys",
|
|
"from io import StringIO",
|
|
"from unittest.mock import patch",
|
|
"from rich.console import Console",
|
|
"from cleveragents.cli.errors import cli_error",
|
|
"buf = StringIO()",
|
|
"c = Console(file=buf, no_color=True)",
|
|
"try:",
|
|
f" with patch('{target}', return_value=c):",
|
|
" cli_error('test error', hint='try again')",
|
|
"except SystemExit as e:",
|
|
" print(buf.getvalue(), end='')",
|
|
" sys.exit(e.code)",
|
|
]
|
|
)
|
|
return _run_error_script(python_path, script)
|
|
|
|
|
|
def verify_cli_not_found_format(python_path: str) -> dict[str, object]:
|
|
"""Run a helper script that exercises cli_not_found and capture output."""
|
|
target = "cleveragents.cli.errors._get_err_console"
|
|
script = "\n".join(
|
|
[
|
|
"import sys",
|
|
"from io import StringIO",
|
|
"from unittest.mock import patch",
|
|
"from rich.console import Console",
|
|
"from cleveragents.cli.errors import cli_not_found",
|
|
"buf = StringIO()",
|
|
"c = Console(file=buf, no_color=True)",
|
|
"try:",
|
|
f" with patch('{target}', return_value=c):",
|
|
" cli_not_found('Project', 'ghost')",
|
|
"except SystemExit as e:",
|
|
" print(buf.getvalue(), end='')",
|
|
" sys.exit(e.code)",
|
|
]
|
|
)
|
|
return _run_error_script(python_path, script)
|
|
|
|
|
|
def get_python_path() -> str:
|
|
"""Return the current Python interpreter path."""
|
|
return sys.executable
|