mngrif
d5af40eefd
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / build (pull_request) Successful in 17s
CI / quality (pull_request) Successful in 20s
CI / security (pull_request) Successful in 28s
CI / typecheck (pull_request) Successful in 37s
CI / integration_tests (pull_request) Failing after 2m15s
CI / unit_tests (pull_request) Successful in 4m34s
CI / docker (pull_request) Successful in 50s
CI / benchmark-regression (pull_request) Successful in 9m9s
CI / coverage (pull_request) Successful in 9m39s
738 lines
23 KiB
Python
738 lines
23 KiB
Python
import json
|
|
import os
|
|
import sys
|
|
import tarfile
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
import nox
|
|
|
|
# Global configuration
|
|
DEFAULT_PYTHON = "3.13"
|
|
SUPPORTED_PYTHONS = ["3.13"]
|
|
nox.options.reuse_existing_virtualenvs = True
|
|
nox.options.error_on_external_run = True
|
|
|
|
BEHAVE_PARALLEL_VERSION = "1.2.4a1"
|
|
BEHAVE_PARALLEL_URL = (
|
|
"https://files.pythonhosted.org/packages/05/9d/22f74dd77bc4fa85d391564a232c49b4e99cfdeac7bfdee8151ea4606632/"
|
|
f"behave-parallel-{BEHAVE_PARALLEL_VERSION}.tar.gz"
|
|
)
|
|
|
|
|
|
def _default_processes() -> int:
|
|
env_override = os.environ.get("TEST_PROCESSES")
|
|
if env_override:
|
|
return int(env_override)
|
|
try:
|
|
cpus = len(os.sched_getaffinity(0)) or 1
|
|
except AttributeError:
|
|
cpus = os.cpu_count() or 1
|
|
return cpus
|
|
|
|
|
|
def _behave_parallel_args(posargs: list[str]) -> list[str]:
|
|
has_custom_processes = any(
|
|
arg in {"--processes", "-j"} or arg.startswith(("--processes=", "-j"))
|
|
for arg in posargs
|
|
)
|
|
if has_custom_processes:
|
|
return []
|
|
return ["--processes", str(_default_processes())]
|
|
|
|
|
|
def _pabot_parallel_args(posargs: list[str]) -> list[str]:
|
|
has_custom_processes = any(
|
|
arg in {"--processes"} or arg.startswith("--processes=") for arg in posargs
|
|
)
|
|
if has_custom_processes:
|
|
return []
|
|
return ["--processes", str(_default_processes())]
|
|
|
|
|
|
def _split_pabot_args(posargs: list[str]) -> tuple[list[str], list[str]]:
|
|
pabot_args: list[str] = []
|
|
robot_args: list[str] = []
|
|
i = 0
|
|
while i < len(posargs):
|
|
arg = posargs[i]
|
|
if arg == "--processes" and i + 1 < len(posargs):
|
|
pabot_args.extend([arg, posargs[i + 1]])
|
|
i += 2
|
|
continue
|
|
if arg.startswith("--processes="):
|
|
pabot_args.append(arg)
|
|
i += 1
|
|
continue
|
|
robot_args.append(arg)
|
|
i += 1
|
|
return pabot_args, robot_args
|
|
|
|
|
|
def _install_behave_parallel(session: nox.Session) -> None:
|
|
"""Install behave-parallel with a custom CLI wrapper."""
|
|
|
|
tmp_dir = Path(session.create_tmp())
|
|
archive_path = tmp_dir / "behave-parallel.tar.gz"
|
|
with (
|
|
urllib.request.urlopen(BEHAVE_PARALLEL_URL) as response,
|
|
archive_path.open("wb") as target,
|
|
):
|
|
target.write(response.read())
|
|
|
|
with tarfile.open(archive_path, "r:gz") as tar:
|
|
tar.extractall(tmp_dir)
|
|
|
|
source_dir = next(tmp_dir.glob("behave-parallel-*/"))
|
|
|
|
# Recreate package with a parallel-aware CLI that aggregates summaries
|
|
pkg_dir = source_dir / "behave_parallel"
|
|
pkg_dir.mkdir(parents=True, exist_ok=True)
|
|
(pkg_dir / "__init__.py").write_text("\n")
|
|
(pkg_dir / "cli.py").write_text(
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, Iterable, List, Tuple
|
|
|
|
DEFAULT_FEATURE_ROOT = "features/"
|
|
SUMMARY_PATTERNS = {
|
|
"features": re.compile(
|
|
r"(\\d+)\\s+feature(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
"scenarios": re.compile(
|
|
r"(\\d+)\\s+scenario(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
"steps": re.compile(
|
|
r"(\\d+)\\s+step(?:s)?\\s+passed,\\s+(\\d+)\\s+failed"
|
|
r"(?:,\\s+(\\d+)\\s+error(?:s)?)?,\\s+(\\d+)\\s+skipped"
|
|
),
|
|
}
|
|
DURATION_PATTERN = re.compile(r"Took\\s+(?:(\\d+)m\\s*)?([\\d\\.]+)s")
|
|
|
|
|
|
def _extract_features_and_args(argv: List[str]) -> Tuple[List[str], List[str]]:
|
|
positional: List[str] = []
|
|
options: List[str] = []
|
|
for arg in argv:
|
|
if arg.startswith("-"):
|
|
options.append(arg)
|
|
else:
|
|
positional.append(arg)
|
|
if positional:
|
|
return positional, options
|
|
return [DEFAULT_FEATURE_ROOT], options
|
|
|
|
|
|
def _empty_summary() -> Dict[str, Dict[str, int] | float]:
|
|
return {
|
|
"features": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"scenarios": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"steps": {"passed": 0, "failed": 0, "errors": 0, "skipped": 0},
|
|
"duration": 0.0,
|
|
}
|
|
|
|
|
|
def _parse_summary(text: str) -> Dict[str, Dict[str, int] | float]:
|
|
summary = _empty_summary()
|
|
for key, pattern in SUMMARY_PATTERNS.items():
|
|
match = pattern.search(text)
|
|
if match:
|
|
summary[key]["passed"] = int(match.group(1))
|
|
summary[key]["failed"] = int(match.group(2))
|
|
summary[key]["errors"] = int(match.group(3) or 0)
|
|
summary[key]["skipped"] = int(match.group(4))
|
|
duration_match = DURATION_PATTERN.search(text)
|
|
if duration_match:
|
|
minutes = duration_match.group(1)
|
|
seconds = float(duration_match.group(2))
|
|
if minutes:
|
|
seconds += int(minutes) * 60
|
|
summary["duration"] = seconds
|
|
return summary
|
|
|
|
|
|
|
|
def _merge_summaries(
|
|
summaries: List[Dict[str, Dict[str, int] | float]],
|
|
) -> Dict[str, Dict[str, int] | float]:
|
|
total = _empty_summary()
|
|
for summary in summaries:
|
|
for bucket in ("features", "scenarios", "steps"):
|
|
for field in ("passed", "failed", "errors", "skipped"):
|
|
total[bucket][field] += summary.get(bucket, {}).get(field, 0)
|
|
total["duration"] += float(summary.get("duration", 0.0))
|
|
return total
|
|
|
|
|
|
def _format_duration(seconds: float) -> str:
|
|
minutes = int(seconds // 60)
|
|
remainder = seconds % 60
|
|
if minutes:
|
|
return f"{minutes}m {remainder:.3f}s"
|
|
return f"{remainder:.3f}s"
|
|
|
|
|
|
def _print_overall_summary(total: Dict[str, Dict[str, int] | float]) -> None:
|
|
print("\\nOverall summary:")
|
|
for bucket in ("features", "scenarios", "steps"):
|
|
b = total[bucket]
|
|
print(
|
|
f"{b['passed']} {bucket} passed, {b['failed']} failed, "
|
|
f"{b['errors']} errored, {b['skipped']} skipped"
|
|
)
|
|
if total["duration"]:
|
|
print(f"Took {_format_duration(float(total['duration']))} (sum across workers)")
|
|
|
|
|
|
|
|
def _run_feature(
|
|
base_args: List[str], feature: str
|
|
) -> Tuple[int, str, str, str, Dict[str, Dict[str, int] | float]]:
|
|
proc = subprocess.run([*base_args, feature], capture_output=True, text=True)
|
|
combined_output = (proc.stdout or "") + "\\n" + (proc.stderr or "")
|
|
summary = _parse_summary(combined_output)
|
|
return proc.returncode, feature, proc.stdout or "", proc.stderr or "", summary
|
|
|
|
|
|
def _build_base_args(other_args: List[str]) -> List[str]:
|
|
if os.environ.get("BEHAVE_PARALLEL_COVERAGE"):
|
|
rcfile = os.environ.get("COVERAGE_RCFILE")
|
|
base_args = [sys.executable, "-m", "coverage", "run", "--parallel-mode"]
|
|
if rcfile:
|
|
base_args.extend(["--rcfile", rcfile])
|
|
base_args.extend(["-m", "behave", *other_args])
|
|
return base_args
|
|
return [sys.executable, "-m", "behave", *other_args]
|
|
|
|
|
|
def _iter_features(paths: Iterable[str]) -> List[str]:
|
|
collected: List[str] = []
|
|
for path in paths:
|
|
p = Path(path)
|
|
if p.is_dir():
|
|
collected.extend(str(fp) for fp in p.rglob("*.feature"))
|
|
else:
|
|
collected.append(str(p))
|
|
return collected
|
|
|
|
|
|
def main(argv: List[str] | None = None) -> None:
|
|
argv = list(sys.argv[1:] if argv is None else argv)
|
|
|
|
parser = argparse.ArgumentParser(add_help=False)
|
|
parser.add_argument("--processes", "-j", type=int, default=None)
|
|
known, remaining = parser.parse_known_args(argv)
|
|
|
|
processes = known.processes or os.cpu_count() or 1
|
|
|
|
feature_args, other_args = _extract_features_and_args(remaining)
|
|
|
|
if any(flag in remaining for flag in ("-h", "--help", "--version")):
|
|
code = subprocess.run(
|
|
[sys.executable, "-m", "behave", *other_args, *feature_args]
|
|
).returncode
|
|
sys.exit(code)
|
|
|
|
feature_paths = _iter_features(feature_args)
|
|
if not feature_paths:
|
|
print("No feature files found", file=sys.stderr)
|
|
sys.exit(0)
|
|
|
|
base_args = _build_base_args(other_args)
|
|
|
|
summaries: List[Dict[str, Dict[str, int] | float]] = []
|
|
failures: List[Tuple[int, str]] = []
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=processes) as executor:
|
|
futures = [
|
|
executor.submit(_run_feature, base_args, feature)
|
|
for feature in feature_paths
|
|
]
|
|
for fut in concurrent.futures.as_completed(futures):
|
|
code, feature, stdout, stderr, summary = fut.result()
|
|
if stdout:
|
|
print(f"=== Output for {feature} ===")
|
|
print(stdout, end="")
|
|
if stderr:
|
|
print(f"=== Stderr for {feature} ===", file=sys.stderr)
|
|
print(stderr, file=sys.stderr, end="")
|
|
summaries.append(summary)
|
|
if code:
|
|
failures.append((code, feature))
|
|
|
|
total = _merge_summaries(summaries)
|
|
_print_overall_summary(total)
|
|
|
|
if failures:
|
|
for code, feature in failures:
|
|
print(f"behave failed for {feature} with code {code}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
"""
|
|
)
|
|
|
|
setup_path = source_dir / "setup.py"
|
|
setup_path.write_text(
|
|
"""
|
|
from setuptools import find_packages, setup
|
|
|
|
setup(
|
|
name="behave-parallel",
|
|
version="1.2.4a1",
|
|
packages=find_packages(),
|
|
include_package_data=True,
|
|
install_requires=["behave>=1.2.6"],
|
|
entry_points={
|
|
"console_scripts": ["behave-parallel=behave_parallel.cli:main"],
|
|
},
|
|
)
|
|
"""
|
|
)
|
|
|
|
session.install("setuptools", "wheel")
|
|
session.install(str(source_dir))
|
|
|
|
|
|
# =============================================================================
|
|
|
|
|
|
# QUICK DEVELOPMENT SESSIONS (Fast - Run daily)
|
|
# =============================================================================
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def lint(session: nox.Session):
|
|
"""Check code formatting and linting."""
|
|
session.install("ruff>=0.15,<0.16")
|
|
session.run("ruff", "check", "src/", "scripts/", "examples/", "features/", "robot/")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def format(session: nox.Session):
|
|
"""Format code with ruff. Pass --check to verify without modifying."""
|
|
session.install("ruff>=0.15,<0.16")
|
|
session.run("ruff", "format", *session.posargs, ".")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def typecheck(session: nox.Session):
|
|
"""Check types with pyright."""
|
|
session.install("pyright")
|
|
session.install("-e", ".")
|
|
session.run("pyright", stderr=sys.stdout)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def unit_tests(session: nox.Session):
|
|
"""Run BDD tests with Behave."""
|
|
session.install("-e", ".[tests]")
|
|
_install_behave_parallel(session)
|
|
|
|
behave_cmd = session.bin + "/behave-parallel"
|
|
parallel_args = _behave_parallel_args(session.posargs)
|
|
|
|
# If a specific feature file is passed, run only that file
|
|
if session.posargs and session.posargs[0].endswith(".feature"):
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
*session.posargs,
|
|
]
|
|
else:
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
"features/",
|
|
*session.posargs,
|
|
]
|
|
|
|
session.env["PYTHONPATH"] = str(Path("src").resolve())
|
|
session.run(*args)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def docs(session: nox.Session):
|
|
"""Build documentation with MkDocs."""
|
|
session.install("-e", ".[docs]")
|
|
# Suppress DeprecationWarning from the third-party `rx` package which calls
|
|
# datetime.datetime.utcfromtimestamp() (deprecated since Python 3.12).
|
|
session.env["PYTHONWARNINGS"] = "ignore::DeprecationWarning:rx"
|
|
session.run("mkdocs", "build")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def serve_docs(session: nox.Session):
|
|
"""Serve docs locally for development."""
|
|
session.install("-e", ".[docs]")
|
|
# Suppress DeprecationWarning from the third-party `rx` package which calls
|
|
# datetime.datetime.utcfromtimestamp() (deprecated since Python 3.12).
|
|
session.env["PYTHONWARNINGS"] = "ignore::DeprecationWarning:rx"
|
|
session.run("mkdocs", "serve", "--dev-addr", "0.0.0.0:8000")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def build(session: nox.Session):
|
|
"""Build the wheel distribution."""
|
|
session.install("build")
|
|
session.run("python", "-m", "build", "--wheel")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def integration_tests(session: nox.Session):
|
|
"""Run Robot Framework integration tests (parallel via pabot).
|
|
|
|
Defaults to conservative parallelism (<=2 processes) to avoid
|
|
resource pressure in CI. Override via PABOT_PROCESSES or by passing
|
|
--processes/--processes=N in session arguments.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
# Override PYTHONPATH so the editable install (src/) for this project
|
|
# is used instead of any inherited PYTHONPATH from the outer environment.
|
|
session.env["PYTHONPATH"] = "src"
|
|
|
|
# Propagate venv bin to PATH so Run Process in robot files finds
|
|
# the venv's python/robot rather than the system copies.
|
|
venv_bin = os.path.join(session.virtualenv.location, "bin")
|
|
session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build/reports/robot", exist_ok=True)
|
|
|
|
# Pass the venv Python path explicitly so Run Process calls use it
|
|
# instead of relying on PATH (which may not propagate to subprocesses).
|
|
venv_python = os.path.join(venv_bin, "python")
|
|
|
|
pabot_args, robot_args = _split_pabot_args(session.posargs)
|
|
parallel_args = _pabot_parallel_args(pabot_args)
|
|
|
|
session.run(
|
|
"pabot",
|
|
*parallel_args,
|
|
*pabot_args,
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"--variable",
|
|
f"PYTHON:{venv_python}",
|
|
"--exclude",
|
|
"slow",
|
|
"--exclude",
|
|
"discovery",
|
|
*robot_args,
|
|
"robot/",
|
|
)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def slow_integration_tests(session: nox.Session):
|
|
"""Run Robot Framework integration tests."""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
session.run(
|
|
"robot",
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"robot/",
|
|
*session.posargs,
|
|
)
|
|
|
|
|
|
COVERAGE_THRESHOLD = 97
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def coverage_report(session: nox.Session):
|
|
"""Generate coverage report from Behave tests.
|
|
|
|
Runs behave tests in parallel via behave-parallel (same approach as
|
|
unit_tests) with each worker collecting coverage independently using
|
|
``coverage run --parallel-mode``. After all workers finish, the
|
|
per-process data files are merged with ``coverage combine``.
|
|
|
|
Coverage threshold is enforced at >=97%.
|
|
|
|
On success, emits: COVERAGE OK: <pct>% (threshold: 97%)
|
|
On failure, emits: COVERAGE FAILED: <pct>% < 97% threshold
|
|
Both are single-line, CI-parseable summary strings.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
_install_behave_parallel(session)
|
|
|
|
os.makedirs("build", exist_ok=True)
|
|
|
|
session.env["PYTHONPATH"] = str(Path("src").resolve())
|
|
session.env["COVERAGE_RCFILE"] = "pyproject.toml"
|
|
session.env["COVERAGE_FILE"] = "build/.coverage"
|
|
# Tell behave-parallel workers to run under coverage --parallel-mode
|
|
session.env["BEHAVE_PARALLEL_COVERAGE"] = "1"
|
|
|
|
# Clean up any existing coverage data
|
|
session.run("coverage", "erase")
|
|
|
|
# Run behave tests in parallel; each worker collects coverage separately
|
|
behave_cmd = session.bin + "/behave-parallel"
|
|
parallel_args = _behave_parallel_args(session.posargs)
|
|
|
|
if session.posargs and session.posargs[0].endswith(".feature"):
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
"--tags=-discovery",
|
|
"--no-capture",
|
|
*parallel_args,
|
|
*session.posargs,
|
|
]
|
|
else:
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
"--tags=-discovery",
|
|
"--no-capture",
|
|
*parallel_args,
|
|
"features/",
|
|
*session.posargs,
|
|
]
|
|
|
|
session.run(*args)
|
|
|
|
# Merge per-worker coverage data files into a single build/.coverage
|
|
session.run("coverage", "combine")
|
|
|
|
# Always generate HTML and XML reports before checking threshold
|
|
session.run("coverage", "html")
|
|
session.run("coverage", "xml")
|
|
|
|
# Generate terminal report and check threshold.
|
|
# coverage report --fail-under exits non-zero if below threshold,
|
|
# but nox intercepts the exit. We run it with success_codes to capture
|
|
# the failure and emit a clear, CI-parseable single-line summary.
|
|
session.run(
|
|
"coverage",
|
|
"report",
|
|
"--show-missing",
|
|
f"--fail-under={COVERAGE_THRESHOLD}",
|
|
success_codes=[0, 2],
|
|
silent=False,
|
|
)
|
|
|
|
# Parse the total percentage from the coverage JSON report for the summary
|
|
total_pct: float = 0.0
|
|
try:
|
|
session.run("coverage", "json", "-o", "build/coverage.json", silent=True)
|
|
with open("build/coverage.json") as f:
|
|
total_pct = json.load(f)["totals"]["percent_covered"]
|
|
except (FileNotFoundError, KeyError, json.JSONDecodeError):
|
|
# Fall back: if json report failed, re-read xml
|
|
total_pct = 0.0
|
|
|
|
rounded_pct = round(total_pct, 1)
|
|
|
|
if rounded_pct >= COVERAGE_THRESHOLD:
|
|
session.log(f"COVERAGE OK: {rounded_pct}% (threshold: {COVERAGE_THRESHOLD}%)")
|
|
else:
|
|
session.error(
|
|
f"COVERAGE FAILED: {rounded_pct}% < {COVERAGE_THRESHOLD}% threshold"
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def pre_commit(session: nox.Session):
|
|
"""Run all pre-commit hooks on all files."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("pre-commit", "run", "--all-files", *session.posargs)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def security_scan(session: nox.Session):
|
|
"""Run security checks matching Forgejo CI: bandit + semgrep + vulture.
|
|
|
|
Steps:
|
|
1. Bandit medium-severity report (non-blocking JSON export to build/)
|
|
2. Bandit high-severity gate (blocks on findings)
|
|
3. Semgrep custom rules from .semgrep.yml (blocks on ERROR-level findings)
|
|
4. Vulture dead-code detection (blocks on confidence >=80%)
|
|
|
|
Severity gates:
|
|
- Bandit HIGH: hard fail (blocks merge)
|
|
- Bandit MEDIUM: report-only (review, do not block)
|
|
- Semgrep ERROR: hard fail (eval/exec/os.system detection)
|
|
- Semgrep WARNING: report-only (pickle.loads)
|
|
- Vulture >=80% confidence: hard fail
|
|
"""
|
|
session.install("-e", ".[dev]")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build", exist_ok=True)
|
|
|
|
# Step 1: Bandit - medium-severity report (non-blocking, mirrors CI JSON export)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"medium",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
"build/bandit-report.json",
|
|
success_codes=[0, 1],
|
|
)
|
|
|
|
# Step 2: Bandit - high-severity gate (blocks on findings)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"high",
|
|
)
|
|
|
|
# Step 3: Semgrep custom rules (blocks on ERROR-severity matches)
|
|
semgrep_config = Path(".semgrep.yml")
|
|
if semgrep_config.exists():
|
|
session.run(
|
|
"semgrep",
|
|
"--config=.semgrep.yml",
|
|
"--error",
|
|
"--quiet",
|
|
"src/",
|
|
success_codes=[0, 1],
|
|
)
|
|
else:
|
|
session.warn("No .semgrep.yml found, skipping semgrep scan")
|
|
|
|
# Step 4: Vulture dead-code detection
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def dead_code(session: nox.Session):
|
|
"""Run vulture to detect dead code."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def complexity(session: nox.Session):
|
|
"""Check code complexity using radon."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"radon",
|
|
"cc",
|
|
"src/cleveragents",
|
|
"--min",
|
|
"C",
|
|
"--show-complexity",
|
|
"--total-average",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def adr_compliance(session: nox.Session):
|
|
"""Check code compliance with Architecture Decision Records."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("python", "scripts/check-adr-compliance.py")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def benchmark(session: nox.Session):
|
|
"""Run Airspeed Velocity benchmarks and publish results."""
|
|
session.install("-e", ".[tests]")
|
|
config_path = "asv.conf.json"
|
|
session.run("asv", "machine", "--yes", f"--config={config_path}")
|
|
session.run(
|
|
"asv",
|
|
"run",
|
|
"--append-samples",
|
|
"--show-stderr",
|
|
"--verbose",
|
|
f"--config={config_path}",
|
|
success_codes=[0, 2],
|
|
)
|
|
session.run("asv", "publish", f"--config={config_path}")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def benchmark_regression(session: nox.Session):
|
|
"""Run Airspeed Velocity benchmarks regression test."""
|
|
session.install("-e", ".[tests]")
|
|
config_path = "asv.conf.json"
|
|
asv_base_sha = os.environ.get("ASV_BASE_SHA")
|
|
session.run("asv", "machine", "--yes", f"--config={config_path}")
|
|
session.run(
|
|
"asv",
|
|
"continuous",
|
|
"--append-samples",
|
|
"--show-stderr",
|
|
"--verbose",
|
|
f"--factor=1.10",
|
|
f"--config={config_path}",
|
|
asv_base_sha,
|
|
"HEAD",
|
|
success_codes=[0, 2],
|
|
)
|
|
|
|
|
|
# Sessions to run by default when running `nox` without arguments
|
|
nox.options.sessions = [
|
|
"lint", # ~5-10 seconds
|
|
"format", # ~5-10 seconds (auto-fixes issues)
|
|
"typecheck", # ~10-20 seconds
|
|
"security_scan", # bandit + vulture (mirrors Forgejo CI security job)
|
|
"dead_code", # vulture dead-code detection
|
|
"unit_tests", # ~30-60 seconds (without ML deps)
|
|
"integration_tests",
|
|
"docs", # ~10-30 seconds
|
|
"build", # ~5-10 seconds
|
|
"benchmark", # ASV benchmarks for performance tracking
|
|
"coverage_report", # ~30-60 seconds (without ML deps)
|
|
]
|