brent.edwards
e8aa5ac268
CI / lint (pull_request) Successful in 15s
CI / typecheck (pull_request) Successful in 28s
CI / security (pull_request) Successful in 21s
CI / quality (pull_request) Successful in 15s
CI / integration_tests (pull_request) Failing after 6m25s
CI / build (pull_request) Successful in 15s
CI / unit_tests (pull_request) Successful in 6m52s
CI / docker (pull_request) Successful in 9s
CI / coverage (pull_request) Successful in 5m5s
- Restore session.env["PATH"] in integration_tests nox session to ensure
Run Process uses venv Python instead of system Python
- Convert bare Resource references to ${CURDIR}/ absolute paths across
30 robot files to fix CI resolution failures
- Add timeout=30s to all Run Process calls in rxpy_route_validation.robot
to prevent hanging tests
- Tag 2 rxpy tests as slow (require running actors unavailable on CI)
- Fix LangGraph test to use correct config file (LANGGRAPH_CONFIG)
- All 204 tests pass (4 excluded: 2 slow + 2 discovery)
582 lines
18 KiB
Python
582 lines
18 KiB
Python
import os
|
|
import sys
|
|
import tarfile
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
import nox
|
|
|
|
# Global configuration
|
|
DEFAULT_PYTHON = "3.13"
|
|
SUPPORTED_PYTHONS = ["3.13"]
|
|
nox.options.reuse_existing_virtualenvs = True
|
|
nox.options.error_on_external_run = True
|
|
|
|
BEHAVE_PARALLEL_VERSION = "1.2.4a1"
|
|
BEHAVE_PARALLEL_URL = (
|
|
"https://files.pythonhosted.org/packages/05/9d/22f74dd77bc4fa85d391564a232c49b4e99cfdeac7bfdee8151ea4606632/"
|
|
f"behave-parallel-{BEHAVE_PARALLEL_VERSION}.tar.gz"
|
|
)
|
|
|
|
|
|
def _default_processes() -> int:
|
|
try:
|
|
return len(os.sched_getaffinity(0)) or 1
|
|
except AttributeError:
|
|
return os.cpu_count() or 1
|
|
|
|
|
|
def _behave_parallel_args(posargs: list[str]) -> list[str]:
|
|
has_custom_processes = any(
|
|
arg in {"--processes", "-j"} or arg.startswith(("--processes=", "-j"))
|
|
for arg in posargs
|
|
)
|
|
if has_custom_processes:
|
|
return []
|
|
return ["--processes", str(_default_processes())]
|
|
|
|
|
|
def _install_behave_parallel(session: nox.Session) -> None:
|
|
"""Install behave-parallel with a custom CLI wrapper."""
|
|
|
|
tmp_dir = Path(session.create_tmp())
|
|
archive_path = tmp_dir / "behave-parallel.tar.gz"
|
|
with (
|
|
urllib.request.urlopen(BEHAVE_PARALLEL_URL) as response,
|
|
archive_path.open("wb") as target,
|
|
):
|
|
target.write(response.read())
|
|
|
|
with tarfile.open(archive_path, "r:gz") as tar:
|
|
tar.extractall(tmp_dir)
|
|
|
|
source_dir = next(tmp_dir.glob("behave-parallel-*/"))
|
|
|
|
# Recreate package with a parallel-aware CLI that aggregates summaries
|
|
pkg_dir = source_dir / "behave_parallel"
|
|
pkg_dir.mkdir(parents=True, exist_ok=True)
|
|
(pkg_dir / "__init__.py").write_text("\n")
|
|
(pkg_dir / "cli.py").write_text(
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, Iterable, List, Tuple
|
|
|
|
DEFAULT_FEATURE_ROOT = "features/"
|
|
SUMMARY_PATTERNS = {
|
|
"features": re.compile(
|
|
r"(\\d+)\\s+feature(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
"scenarios": re.compile(
|
|
r"(\\d+)\\s+scenario(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
"steps": re.compile(
|
|
r"(\\d+)\\s+step(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
}
|
|
DURATION_PATTERN = re.compile(r"Took\\s+(?:(\\d+)m\\s*)?([\\d\\.]+)s")
|
|
|
|
|
|
def _extract_features_and_args(argv: List[str]) -> Tuple[List[str], List[str]]:
|
|
positional: List[str] = []
|
|
options: List[str] = []
|
|
for arg in argv:
|
|
if arg.startswith("-"):
|
|
options.append(arg)
|
|
else:
|
|
positional.append(arg)
|
|
if positional:
|
|
return positional, options
|
|
return [DEFAULT_FEATURE_ROOT], options
|
|
|
|
|
|
def _empty_summary() -> Dict[str, Dict[str, int] | float]:
|
|
return {
|
|
"features": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"scenarios": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"steps": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"duration": 0.0,
|
|
}
|
|
|
|
|
|
def _parse_summary(text: str) -> Dict[str, Dict[str, int] | float]:
|
|
summary = _empty_summary()
|
|
for key, pattern in SUMMARY_PATTERNS.items():
|
|
match = pattern.search(text)
|
|
if match:
|
|
summary[key]["passed"] = int(match.group(1))
|
|
summary[key]["failed"] = int(match.group(2))
|
|
summary[key]["errors"] = int(match.group(3) or 0)
|
|
summary[key]["skipped"] = int(match.group(4))
|
|
duration_match = DURATION_PATTERN.search(text)
|
|
if duration_match:
|
|
minutes = duration_match.group(1)
|
|
seconds = float(duration_match.group(2))
|
|
if minutes:
|
|
seconds += int(minutes) * 60
|
|
summary["duration"] = seconds
|
|
return summary
|
|
|
|
|
|
|
|
def _merge_summaries(
|
|
summaries: List[Dict[str, Dict[str, int] | float]],
|
|
) -> Dict[str, Dict[str, int] | float]:
|
|
total = _empty_summary()
|
|
for summary in summaries:
|
|
for bucket in ("features", "scenarios", "steps"):
|
|
for field in ("passed", "failed", "errors", "skipped"):
|
|
total[bucket][field] += summary.get(bucket, {}).get(field, 0)
|
|
total["duration"] += float(summary.get("duration", 0.0))
|
|
return total
|
|
|
|
|
|
def _format_duration(seconds: float) -> str:
|
|
minutes = int(seconds // 60)
|
|
remainder = seconds % 60
|
|
if minutes:
|
|
return f"{minutes}m {remainder:.3f}s"
|
|
return f"{remainder:.3f}s"
|
|
|
|
|
|
def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None:
|
|
print("\\nOverall summary:")
|
|
for bucket in ("features", "scenarios", "steps"):
|
|
b = total[bucket]
|
|
print(
|
|
f"{b['passed']} {bucket} passed, {b['failed']} failed, "
|
|
f"{b['errors']} errored, {b['skipped']} skipped"
|
|
)
|
|
if total["duration"]:
|
|
print(f"Took {_format_duration(float(total['duration']))} (sum across workers)")
|
|
|
|
|
|
|
|
def _run_feature(
|
|
base_args: List[str], feature: str
|
|
) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]:
|
|
proc = subprocess.run([*base_args, feature], capture_output=True, text=True)
|
|
combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "")
|
|
summary = _parse_summary(combined_output)
|
|
return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary
|
|
|
|
|
|
def _build_base_args(other_args: List[str]) -> List[str]:
|
|
if os.environ.get("BEHAVE_PARALLEL_COVERAGE"):
|
|
rcfile = os.environ.get("COVERAGE_RCFILE")
|
|
base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"]
|
|
if rcfile:
|
|
base_args.extend(["--rcfile", rcfile])
|
|
base_args.extend(["-m", "behave", *other_args])
|
|
return base_args
|
|
return [sys.executable, "-m", "behave", *other_args]
|
|
|
|
|
|
def _iter_features(paths: Iterable[str]) -> List[str]:
|
|
collected: List[str] = []
|
|
for path in paths:
|
|
p = Path(path)
|
|
if p.is_dir():
|
|
collected.extend(str(fp) for fp in p.rglob("*.feature"))
|
|
else:
|
|
collected.append(str(p))
|
|
return collected
|
|
|
|
|
|
def main(argv: List[str] | None = None) -> None:
|
|
argv = list(sys.argv[1:] if argv is None else argv)
|
|
|
|
parser = argparse.ArgumentParser(add_help=False)
|
|
parser.add_argument("--processes", "-j", type=int, default=None)
|
|
known, remaining = parser.parse_known_args(argv)
|
|
|
|
processes = known.processes or os.cpu_count() or 1
|
|
|
|
feature_args, other_args = _extract_features_and_args(remaining)
|
|
|
|
if any(flag in remaining for flag in ("-h", "--help", "--version")):
|
|
code = subprocess.run(
|
|
[sys.executable, "-m", "behave", *other_args, *feature_args]
|
|
).returncode
|
|
sys.exit(code)
|
|
|
|
feature_paths = _iter_features(feature_args)
|
|
if not feature_paths:
|
|
print("No feature files found", file=sys.stderr)
|
|
sys.exit(0)
|
|
|
|
base_args = _build_base_args(other_args)
|
|
|
|
summaries: List[Dict[str, Dict[str, int] | float]] = []
|
|
failures: List[Tuple[int, str]] = []
|
|
with concurrent.futures.ProcessPoolExecutor(max_workers=processes) as executor:
|
|
futures = [
|
|
executor.submit(_run_feature, base_args, feature)
|
|
for feature in feature_paths
|
|
]
|
|
for fut in concurrent.futures.as_completed(futures):
|
|
code, feature, stdout, stderr, summary = fut.result()
|
|
if stdout:
|
|
print(f"=== Output for {feature} ===")
|
|
print(stdout, end="")
|
|
if stderr:
|
|
print(f"=== Stderr for {feature} ===", file=sys.stderr)
|
|
print(stderr, file=sys.stderr, end="")
|
|
summaries.append(summary)
|
|
if code:
|
|
failures.append((code, feature))
|
|
|
|
total = _merge_summaries(summaries)
|
|
_print_overall_summary(total)
|
|
|
|
if failures:
|
|
for code, feature in failures:
|
|
print(f"behave failed for {feature} with code {code}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
"""
|
|
)
|
|
|
|
setup_path = source_dir / "setup.py"
|
|
setup_path.write_text(
|
|
"""
|
|
from setuptools import find_packages, setup
|
|
|
|
setup(
|
|
name="behave-parallel",
|
|
version="1.2.4a1",
|
|
packages=find_packages(),
|
|
include_package_data=True,
|
|
install_requires=["behave>=1.2.6"],
|
|
entry_points={
|
|
"console_scripts": ["behave-parallel=behave_parallel.cli:main"],
|
|
},
|
|
)
|
|
"""
|
|
)
|
|
|
|
session.install("setuptools", "wheel")
|
|
session.install(str(source_dir))
|
|
|
|
|
|
# =============================================================================
|
|
|
|
|
|
# QUICK DEVELOPMENT SESSIONS (Fast - Run daily)
|
|
# =============================================================================
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def lint(session: nox.Session):
|
|
"""Check code formatting and linting."""
|
|
session.install("ruff")
|
|
session.run("ruff", "check", "src/", "scripts/", "examples/", "features/", "robot/")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def format(session: nox.Session):
|
|
"""Format code with ruff. Pass --check to verify without modifying."""
|
|
session.install("ruff")
|
|
session.run("ruff", "format", *session.posargs, ".")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def typecheck(session: nox.Session):
|
|
"""Check types with pyright."""
|
|
session.install("pyright")
|
|
session.install("-e", ".")
|
|
session.run("pyright", stderr=sys.stdout)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def unit_tests(session: nox.Session):
|
|
"""Run BDD tests with Behave."""
|
|
session.install("-e", ".[tests]")
|
|
_install_behave_parallel(session)
|
|
|
|
behave_cmd = session.bin + "/behave-parallel"
|
|
parallel_args = _behave_parallel_args(session.posargs)
|
|
|
|
# If a specific feature file is passed, run only that file
|
|
if session.posargs and session.posargs[0].endswith(".feature"):
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
*session.posargs,
|
|
]
|
|
else:
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
"features/",
|
|
*session.posargs,
|
|
]
|
|
|
|
session.env["PYTHONPATH"] = "src:."
|
|
session.run(*args)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def docs(session: nox.Session):
|
|
"""Build documentation with MkDocs."""
|
|
session.install("-e", ".[docs]")
|
|
session.run("mkdocs", "build")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def serve_docs(session: nox.Session):
|
|
"""Serve docs locally for development."""
|
|
session.install("-e", ".[docs]")
|
|
session.run("mkdocs", "serve", "--dev-addr", "0.0.0.0:8000")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def build(session: nox.Session):
|
|
"""Build the wheel distribution."""
|
|
session.install("build")
|
|
session.run("python", "-m", "build", "--wheel")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def integration_tests(session: nox.Session):
|
|
"""Run Robot Framework integration tests (sequential via robot).
|
|
|
|
Uses ``robot`` directly instead of ``pabot`` for reliable execution
|
|
in constrained CI containers. pabot spawns a new ``robot``
|
|
subprocess per suite which exhausts process/FD limits on CI after
|
|
~24 suites, causing ``FileNotFoundError: 'robot'``.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
|
|
# Propagate venv bin to PATH so Run Process in robot files finds
|
|
# the venv's python/robot rather than the system copies.
|
|
venv_bin = os.path.join(session.virtualenv.location, "bin")
|
|
session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "")
|
|
|
|
# Debug: confirm robot is discoverable and show working directory
|
|
session.run(
|
|
"python",
|
|
"-c",
|
|
"import shutil, os; print('robot at:', shutil.which('robot')); "
|
|
"print('cwd:', os.getcwd()); "
|
|
"print('robot/ contents:', sorted(os.listdir('robot')))",
|
|
)
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build/reports/robot", exist_ok=True)
|
|
|
|
session.run(
|
|
"robot",
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"--exclude",
|
|
"slow",
|
|
"--exclude",
|
|
"discovery",
|
|
*session.posargs,
|
|
"robot/",
|
|
)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def slow_integration_tests(session: nox.Session):
|
|
"""Run Robot Framework integration tests."""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
session.run(
|
|
"robot",
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"robot/",
|
|
*session.posargs,
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def coverage_report(session: nox.Session):
|
|
"""Generate coverage report from Behave tests.
|
|
|
|
Runs behave under coverage in serial mode to ensure accurate data
|
|
collection. Coverage threshold is enforced at >=97%.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
|
|
session.env["PYTHONPATH"] = "src:."
|
|
session.env["COVERAGE_RCFILE"] = "pyproject.toml"
|
|
session.env["COVERAGE_FILE"] = "build/.coverage"
|
|
|
|
# Clean up any existing coverage data
|
|
session.run("coverage", "erase")
|
|
|
|
# Run behave under coverage (serial mode for reliable data collection)
|
|
behave_args = [
|
|
"coverage",
|
|
"run",
|
|
"--source=src",
|
|
"-m",
|
|
"behave",
|
|
"-q",
|
|
"--tags=-discovery",
|
|
"--no-capture",
|
|
]
|
|
|
|
if session.posargs and session.posargs[0].endswith(".feature"):
|
|
behave_args.extend(session.posargs)
|
|
else:
|
|
behave_args.append("features/")
|
|
behave_args.extend(session.posargs)
|
|
|
|
session.run(*behave_args)
|
|
|
|
# Generate and display coverage reports
|
|
session.run("coverage", "report", "--show-missing", "--fail-under=97")
|
|
session.run("coverage", "html")
|
|
session.run("coverage", "xml")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def pre_commit(session: nox.Session):
|
|
"""Run all pre-commit hooks on all files."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("pre-commit", "run", "--all-files", *session.posargs)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def security_scan(session: nox.Session):
|
|
"""Run security checks matching Forgejo CI: bandit scan + vulture dead code."""
|
|
session.install("-e", ".[dev]")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build", exist_ok=True)
|
|
|
|
# Step 1: Bandit - medium-severity report (non-blocking, mirrors CI JSON export)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"medium",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
"build/bandit-report.json",
|
|
success_codes=[0, 1],
|
|
)
|
|
|
|
# Step 2: Bandit - high-severity gate (blocks on findings)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"high",
|
|
)
|
|
|
|
# Step 3: Vulture dead-code detection
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def dead_code(session: nox.Session):
|
|
"""Run vulture to detect dead code."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def complexity(session: nox.Session):
|
|
"""Check code complexity using radon."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"radon",
|
|
"cc",
|
|
"src/cleveragents",
|
|
"--min",
|
|
"C",
|
|
"--show-complexity",
|
|
"--total-average",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def adr_compliance(session: nox.Session):
|
|
"""Check code compliance with Architecture Decision Records."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("python", "scripts/check-adr-compliance.py")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def benchmark(session: nox.Session):
|
|
"""Run Airspeed Velocity benchmarks and publish results."""
|
|
session.install("-e", ".[tests]")
|
|
config_path = "asv.conf.json"
|
|
session.run("asv", "machine", "--yes", f"--config={config_path}")
|
|
session.run("asv", "run", "--show-stderr", "--verbose", f"--config={config_path}")
|
|
session.run("asv", "publish", f"--config={config_path}")
|
|
|
|
|
|
# Sessions to run by default when running `nox` without arguments
|
|
nox.options.sessions = [
|
|
"lint", # ~5-10 seconds
|
|
"format", # ~5-10 seconds (auto-fixes issues)
|
|
"typecheck", # ~10-20 seconds
|
|
"security_scan", # bandit + vulture (mirrors Forgejo CI security job)
|
|
"dead_code", # vulture dead-code detection
|
|
"unit_tests", # ~30-60 seconds (without ML deps)
|
|
"integration_tests",
|
|
"docs", # ~10-30 seconds
|
|
"build", # ~5-10 seconds
|
|
"benchmark", # ASV benchmarks for performance tracking
|
|
"coverage_report", # ~30-60 seconds (without ML deps)
|
|
]
|