import contextlib import os import sys import tarfile import urllib.request from pathlib import Path import nox # Global configuration DEFAULT_PYTHON = "3.13" SUPPORTED_PYTHONS = ["3.13"] nox.options.reuse_existing_virtualenvs = True nox.options.error_on_external_run = True BEHAVE_PARALLEL_VERSION = "1.2.4a1" BEHAVE_PARALLEL_URL = ( "https://files.pythonhosted.org/packages/05/9d/22f74dd77bc4fa85d391564a232c49b4e99cfdeac7bfdee8151ea4606632/" f"behave-parallel-{BEHAVE_PARALLEL_VERSION}.tar.gz" ) def _default_processes() -> int: try: return len(os.sched_getaffinity(0)) or 1 except AttributeError: return os.cpu_count() or 1 def _behave_parallel_args(posargs: list[str]) -> list[str]: has_custom_processes = any( arg in {"--processes", "-j"} or arg.startswith(("--processes=", "-j")) for arg in posargs ) if has_custom_processes: return [] return ["--processes", str(_default_processes())] def _pabot_parallel_args(posargs: list[str]) -> list[str]: has_custom_processes = any( arg in {"--processes", "-p"} or arg.startswith(("--processes=", "-p")) for arg in posargs ) if has_custom_processes: return [] return ["--processes", str(_default_processes())] def _install_behave_parallel(session: nox.Session) -> None: """Install behave-parallel with a custom CLI wrapper.""" tmp_dir = Path(session.create_tmp()) archive_path = tmp_dir / "behave-parallel.tar.gz" with ( urllib.request.urlopen(BEHAVE_PARALLEL_URL) as response, archive_path.open("wb") as target, ): target.write(response.read()) with tarfile.open(archive_path, "r:gz") as tar: tar.extractall(tmp_dir) source_dir = next(tmp_dir.glob("behave-parallel-*/")) # Recreate package with a parallel-aware CLI that aggregates summaries pkg_dir = source_dir / "behave_parallel" pkg_dir.mkdir(parents=True, exist_ok=True) (pkg_dir / "__init__.py").write_text("\n") (pkg_dir / "cli.py").write_text( """ import argparse import concurrent.futures import os import re import subprocess import sys from pathlib import Path from typing import Dict, Iterable, List, Tuple DEFAULT_FEATURE_ROOT = "features/" SUMMARY_PATTERNS = { "features": re.compile(r"(\\d+)\\s+feature(?:s)?\\s+passed,\\s+(\\d+)\\s+failed(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"), "scenarios": re.compile(r"(\\d+)\\s+scenario(?:s)?\\s+passed,\\s+(\\d+)\\s+failed(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"), "steps": re.compile(r"(\\d+)\\s+step(?:s)?\\s+passed,\\s+(\\d+)\\s+failed(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"), } DURATION_PATTERN = re.compile(r"Took\\s+(?:(\\d+)m\\s*)?([\\d\\.]+)s") def _extract_features_and_args(argv: List[str]) -> Tuple[List[str], List[str]]: positional: List[str] = [] options: List[str] = [] for arg in argv: if arg.startswith("-"): options.append(arg) else: positional.append(arg) if positional: return positional, options return [DEFAULT_FEATURE_ROOT], options def _empty_summary() -> Dict[str, Dict[str, int] | float]: return { "features": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0}, "scenarios": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0}, "steps": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0}, "duration": 0.0, } def _parse_summary(text: str) -> Dict[str, Dict[str, int] | float]: summary = _empty_summary() for key, pattern in SUMMARY_PATTERNS.items(): match = pattern.search(text) if match: summary[key]["passed"] = int(match.group(1)) summary[key]["failed"] = int(match.group(2)) summary[key]["errors"] = int(match.group(3) or 0) summary[key]["skipped"] = int(match.group(4)) duration_match = DURATION_PATTERN.search(text) if duration_match: minutes = duration_match.group(1) seconds = float(duration_match.group(2)) if minutes: seconds += int(minutes) * 60 summary["duration"] = seconds return summary def _merge_summaries(summaries: List[Dict[str, Dict[str, int] | float]]) -> Dict[str, Dict[str, int] | float]: total = _empty_summary() for summary in summaries: for bucket in ("features", "scenarios", "steps"): for field in ("passed", "failed", "errors", "skipped"): total[bucket][field] += summary.get(bucket, {}).get(field, 0) total["duration"] += float(summary.get("duration", 0.0)) return total def _format_duration(seconds: float) -> str: minutes = int(seconds // 60) remainder = seconds % 60 if minutes: return f"{minutes}m {remainder:.3f}s" return f"{remainder:.3f}s" def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None: print("\\nOverall summary:") print( f"{total['features']['passed']} features passed, {total['features']['failed']} failed, {total['features']['errors']} errored, {total['features']['skipped']} skipped" ) print( f"{total['scenarios']['passed']} scenarios passed, {total['scenarios']['failed']} failed, {total['scenarios']['errors']} errored, {total['scenarios']['skipped']} skipped" ) print( f"{total['steps']['passed']} steps passed, {total['steps']['failed']} failed, {total['steps']['errors']} errored, {total['steps']['skipped']} skipped" ) if total["duration"]: print(f"Took {_format_duration(float(total['duration']))} (sum across workers)") def _run_feature(base_args: List[str], feature: str) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]: proc = subprocess.run([*base_args, feature], capture_output=True, text=True) combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "") summary = _parse_summary(combined_output) return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary def _build_base_args(other_args: List[str]) -> List[str]: if os.environ.get("BEHAVE_PARALLEL_COVERAGE"): rcfile = os.environ.get("COVERAGE_RCFILE") base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"] if rcfile: base_args.extend(["--rcfile", rcfile]) base_args.extend(["-m", "behave", *other_args]) return base_args return [sys.executable, "-m", "behave", *other_args] def _iter_features(paths: Iterable[str]) -> List[str]: collected: List[str] = [] for path in paths: p = Path(path) if p.is_dir(): collected.extend(str(fp) for fp in p.rglob("*.feature")) else: collected.append(str(p)) return collected def main(argv: List[str] | None = None) -> None: argv = list(sys.argv[1:] if argv is None else argv) parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--processes", "-j", type=int, default=None) known, remaining = parser.parse_known_args(argv) processes = known.processes or os.cpu_count() or 1 feature_args, other_args = _extract_features_and_args(remaining) if any(flag in remaining for flag in ("-h", "--help", "--version")): code = subprocess.run([sys.executable, "-m", "behave", *other_args, *feature_args]).returncode sys.exit(code) feature_paths = _iter_features(feature_args) if not feature_paths: print("No feature files found", file=sys.stderr) sys.exit(0) base_args = _build_base_args(other_args) summaries: List[Dict[str, Dict[str, int] | float]] = [] failures: List[Tuple[int, str]] = [] with concurrent.futures.ProcessPoolExecutor(max_workers=processes) as executor: futures = [executor.submit(_run_feature, base_args, feature) for feature in feature_paths] for fut in concurrent.futures.as_completed(futures): code, feature, stdout, stderr, summary = fut.result() if stdout: print(f"=== Output for {feature} ===") print(stdout, end="") if stderr: print(f"=== Stderr for {feature} ===", file=sys.stderr) print(stderr, file=sys.stderr, end="") summaries.append(summary) if code: failures.append((code, feature)) total = _merge_summaries(summaries) _print_overall_summary(total) if failures: for code, feature in failures: print(f"behave failed for {feature} with code {code}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main() """ ) setup_path = source_dir / "setup.py" setup_path.write_text( """ from setuptools import find_packages, setup setup( name="behave-parallel", version="1.2.4a1", packages=find_packages(), include_package_data=True, install_requires=["behave>=1.2.6"], entry_points={ "console_scripts": ["behave-parallel=behave_parallel.cli:main"], }, ) """ ) session.install("setuptools", "wheel") session.install(str(source_dir)) # ============================================================================= # QUICK DEVELOPMENT SESSIONS (Fast - Run daily) # ============================================================================= @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def lint(session: nox.Session): """Check code formatting and linting.""" session.install("ruff") session.run("ruff", "check", "src/", "scripts/", "examples/") @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def format(session: nox.Session): """Format code with ruff.""" session.install("ruff") session.run("ruff", "format", ".") # session.run("ruff", "check", "--fix", ".") @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def typecheck(session: nox.Session): """Check types with pyright.""" session.install("pyright") session.install("-e", ".") session.run("pyright", stderr=sys.stdout) @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def unit_tests(session: nox.Session): """Run BDD tests with Behave.""" session.install("-e", ".[tests]") _install_behave_parallel(session) behave_cmd = session.bin + "/behave-parallel" parallel_args = _behave_parallel_args(session.posargs) # If a specific feature file is passed, run only that file if session.posargs and session.posargs[0].endswith(".feature"): args = [ behave_cmd, "-q", "--tags=-discovery", *parallel_args, *session.posargs, ] else: args = [ behave_cmd, "-q", "--tags=-discovery", *parallel_args, "features/", *session.posargs, ] session.env["PYTHONPATH"] = "src:." session.run(*args) @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def docs(session: nox.Session): """Build documentation with MkDocs.""" session.install("-e", ".[docs]") session.run("mkdocs", "build") @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def serve_docs(session: nox.Session): """Serve docs locally for development.""" session.install("-e", ".[docs]") session.run("mkdocs", "serve", "--dev-addr", "0.0.0.0:8000") @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def build(session: nox.Session): """Build the wheel distribution.""" session.install("build") session.run("python", "-m", "build", "--wheel") @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def integration_tests(session: nox.Session): """Run Robot Framework integration tests (excluding discovery tests).""" session.install("-e", ".[tests]") session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" parallel_args = _pabot_parallel_args(session.posargs) common_args = [ "pabot", "--outputdir", "build", "--loglevel", "INFO", "--report", "reports/robot/report.html", "--log", "reports/robot/log.html", "--xunit", "reports/robot/xunit.xml", "--exclude", "slow", "--exclude", "discovery", *parallel_args, ] if session.posargs: session.run( *common_args, "--exitonfailure", # Exit immediately on test failure *session.posargs, external=False, # Ensure exit code is checked success_codes=[0], # Only code 0 is success ) else: # Default: run all tests in robot/ directory session.run( *common_args, "robot/", external=False, # Ensure exit code is checked success_codes=[0], # Only code 0 is success ) @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def slow_integration_tests(session: nox.Session): """Run Robot Framework integration tests.""" session.install("-e", ".[tests]") session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true" session.run( "robot", "--outputdir", "build/reports/robot", "--loglevel", "INFO", "--report", "report.html", "--log", "log.html", "--xunit", "xunit.xml", "robot/", *session.posargs, ) @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def coverage_report(session: nox.Session): """Generate coverage report from Behave tests.""" session.install("-e", ".[tests]") _install_behave_parallel(session) # Ensure coverage data writes to the configured location session.env["PYTHONPATH"] = "src:." session.env["BEHAVE_PARALLEL_COVERAGE"] = "1" session.env["COVERAGE_RCFILE"] = "pyproject.toml" session.env["COVERAGE_FILE"] = "build/.coverage" # Clean up any existing coverage data session.run("coverage", "erase") behave_cmd = session.bin + "/behave-parallel" parallel_args = _behave_parallel_args(session.posargs) if session.posargs and session.posargs[0].endswith(".feature"): args = [ behave_cmd, "-q", "--tags=not @discovery", *parallel_args, *session.posargs, ] else: args = [ behave_cmd, "-q", "--tags=not @discovery", *parallel_args, "features/", *session.posargs, ] session.run(*args) # Combine parallel coverage data produced by each worker with contextlib.suppress(Exception): session.run("coverage", "combine") # Generate and display coverage reports session.run("coverage", "report", "--show-missing", "--fail-under=85") session.run("coverage", "html") session.run("coverage", "xml") @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def discovery_tests(session: nox.Session): """Run Phase 0 discovery tests (tagged with @discovery).""" session.install("-e", ".[tests]") _install_behave_parallel(session) behave_cmd = session.bin + "/behave-parallel" parallel_args = _behave_parallel_args(session.posargs) # If a specific feature file is passed, run only that file if session.posargs and session.posargs[0].endswith(".feature"): session.run( behave_cmd, "-q", "--tags=@discovery", *parallel_args, *session.posargs, ) return session.run( behave_cmd, "-q", "--tags=@discovery", *parallel_args, "features/", *session.posargs, ) @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def discovery_coverage(session: nox.Session): """Generate coverage report for discovery tests.""" session.install("-e", ".[tests]") # Clean up any existing coverage data session.run("coverage", "erase") # Run only discovery tests with coverage (sequential) session.run( "coverage", "run", "--source=src/cleveragents/discovery", "-m", "behave", "-q", "--tags=@discovery", "features/", *session.posargs, ) # Combine parallel coverage data (required when parallel=true in config) with contextlib.suppress(Exception): session.run("coverage", "combine") # No parallel data to combine # Generate and display coverage reports session.run("coverage", "report", "--show-missing") session.run("coverage", "html") session.run("coverage", "xml") @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def discovery_integration(session: nox.Session): """Run Robot Framework integration tests tagged with 'discovery'.""" session.install("-e", ".[tests]") # If posargs provided, run only those specific files/suites with discovery tag if session.posargs: session.run( "robot", "--outputdir", "build/reports/robot_discovery", "--loglevel", "INFO", "--report", "discovery_report.html", "--log", "discovery_log.html", "--xunit", "discovery_xunit.xml", "--include", "discovery", # Include only discovery tests "--exitonfailure", # Exit immediately on test failure *session.posargs, external=False, # Ensure exit code is checked success_codes=[0], # Only code 0 is success ) else: # Default: run all discovery tests in robot/ directory session.run( "robot", "--outputdir", "build/reports/robot_discovery", "--loglevel", "INFO", "--report", "discovery_report.html", "--log", "discovery_log.html", "--xunit", "discovery_xunit.xml", "--include", "discovery", # Include only discovery tests "robot/", external=False, # Ensure exit code is checked success_codes=[0], # Only code 0 is success ) @nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv") def benchmark(session: nox.Session): """Run Airspeed Velocity benchmarks and publish results.""" session.install("-e", ".[tests]") config_path = "asv.conf.json" session.run("asv", "run", "--show-stderr", "--verbose", f"--config={config_path}") session.run("asv", "publish", f"--config={config_path}") # Sessions to run by default when running `nox` without arguments nox.options.sessions = [ "lint", # ~5-10 seconds "format", # ~5-10 seconds (auto-fixes issues) "typecheck", # ~10-20 seconds "unit_tests", # ~30-60 seconds (without ML deps) "integration_tests", "docs", # ~10-30 seconds "build", # ~5-10 seconds "benchmark", # ASV benchmarks for performance tracking "coverage_report", # ~30-60 seconds (without ML deps) ]