HAL9000
747513bc59
Fix broken escape hatch mechanism and address reviewer feedback: - Replace non-functional comment-based pattern-not clauses with Semgrep's native # nosemgrep mechanism for the escape hatch. Semgrep strips comments from the AST so pattern-not clauses matching inline comments never fire; # nosemgrep is the only reliable per-line suppression mechanism. - Require both # nosemgrep: <rule-id> AND # error-propagation: allow on the same line: the former is the actual suppression, the latter is the mandatory human-readable audit annotation. - Add raise $EXC from $CAUSE pattern-not entries for both Exception and BaseException variants to prevent false positives on legitimate exception chaining (raise ServiceError from e). - Switch nox -s lint Semgrep invocation to audit mode (no --error) for the phased rollout: the codebase has ~337 existing suppressions that must be triaged before enforcement mode is enabled. A comment in noxfile.py documents the migration path and references #9103. - Update CONTRIBUTING.md examples and enforcement description to reflect the dual-comment requirement.
1262 lines
44 KiB
Python
1262 lines
44 KiB
Python
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import tomllib
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
import nox
|
|
|
|
# Global configuration
|
|
DEFAULT_PYTHON = "3.13"
|
|
SUPPORTED_PYTHONS = ["3.13"]
|
|
nox.options.reuse_existing_virtualenvs = True
|
|
nox.options.error_on_external_run = True
|
|
|
|
BEHAVE_PARALLEL_VERSION = "2.0.0"
|
|
|
|
|
|
def _default_processes() -> int:
|
|
env_override = os.environ.get("TEST_PROCESSES")
|
|
if env_override:
|
|
return int(env_override)
|
|
try:
|
|
cpus = len(os.sched_getaffinity(0)) or 1
|
|
except AttributeError:
|
|
cpus = os.cpu_count() or 1
|
|
# Keep default parallelism conservative to avoid timeout/OOM flakes
|
|
# under heavy Robot/pabot subprocess fan-out in CI and shared runners.
|
|
# Callers can still override with TEST_PROCESSES / --processes.
|
|
return cpus
|
|
|
|
|
|
def _behave_parallel_args(posargs: list[str]) -> list[str]:
|
|
has_custom_processes = any(
|
|
arg in {"--processes", "-j"} or arg.startswith(("--processes=", "-j"))
|
|
for arg in posargs
|
|
)
|
|
if has_custom_processes:
|
|
return []
|
|
return ["--processes", str(_default_processes())]
|
|
|
|
|
|
def _pabot_parallel_args(posargs: list[str]) -> list[str]:
|
|
has_custom_processes = any(
|
|
arg in {"--processes"} or arg.startswith("--processes=") for arg in posargs
|
|
)
|
|
if has_custom_processes:
|
|
return []
|
|
# Integration tests are significantly heavier than unit tests and can
|
|
# become unstable on shared runners when pabot fans out aggressively.
|
|
# Cap parallelism at 6: at the prior <=2 the ~47min suite serialized to
|
|
# ~24min wall on a 32-core runner (2x speedup == 2 lanes); 6 lanes cut
|
|
# that to ~6-8min while staying well short of the per-core fan-out that
|
|
# triggers LLM 429 rate-limits / OOM flakes. Override via
|
|
# TEST_PROCESSES / --processes.
|
|
pabot_default = min(6, _default_processes())
|
|
return ["--processes", str(pabot_default)]
|
|
|
|
|
|
def _split_pabot_args(posargs: list[str]) -> tuple[list[str], list[str]]:
|
|
pabot_args: list[str] = []
|
|
robot_args: list[str] = []
|
|
i = 0
|
|
while i < len(posargs):
|
|
arg = posargs[i]
|
|
if arg == "--processes" and i + 1 < len(posargs):
|
|
pabot_args.extend([arg, posargs[i + 1]])
|
|
i += 2
|
|
continue
|
|
if arg.startswith("--processes="):
|
|
pabot_args.append(arg)
|
|
i += 1
|
|
continue
|
|
robot_args.append(arg)
|
|
i += 1
|
|
return pabot_args, robot_args
|
|
|
|
|
|
def _create_template_db(session: nox.Session) -> str:
|
|
"""Build the pre-migrated SQLite template and return its path.
|
|
|
|
Runs ``scripts/create_template_db.py`` to produce a SQLite file with all
|
|
tables already created via ``Base.metadata.create_all()`` and the
|
|
alembic_version table stamped at HEAD. Each test scenario can then
|
|
``shutil.copy`` this file instead of running 25 Alembic migrations.
|
|
"""
|
|
template_path = str(Path("build/.template-migrated.db").resolve())
|
|
session.run(
|
|
"python",
|
|
"scripts/create_template_db.py",
|
|
template_path,
|
|
silent=True,
|
|
)
|
|
return template_path
|
|
|
|
|
|
def _install_behave_parallel(session: nox.Session) -> None:
|
|
"""Install behave-parallel with an in-process parallel runner.
|
|
|
|
Instead of spawning one subprocess per feature file (the old model),
|
|
the new CLI runs features in-process via behave's Python API. Step
|
|
definitions and environment hooks are loaded once; each feature file
|
|
is then parsed and executed without interpreter startup overhead.
|
|
|
|
Parallel execution uses ``multiprocessing.Pool`` with the ``fork``
|
|
start method so that already-imported modules are shared (copy-on-write)
|
|
across workers.
|
|
|
|
The runner script is read from ``scripts/run_behave_parallel.py`` so
|
|
that the noxfile stays concise and the script can be linted and typed
|
|
independently.
|
|
"""
|
|
tmp_dir = Path(session.create_tmp())
|
|
source_dir = tmp_dir / "behave-parallel-inprocess"
|
|
source_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
pkg_dir = source_dir / "behave_parallel"
|
|
pkg_dir.mkdir(parents=True, exist_ok=True)
|
|
(pkg_dir / "__init__.py").write_text("\n")
|
|
|
|
runner_script = Path(__file__).parent / "scripts" / "run_behave_parallel.py"
|
|
(pkg_dir / "cli.py").write_text(runner_script.read_text(encoding="utf-8"))
|
|
|
|
formatter_script = (
|
|
Path(__file__).parent / "scripts" / "behave_pass_suppress_formatter.py"
|
|
)
|
|
(pkg_dir / "behave_pass_suppress_formatter.py").write_text(
|
|
formatter_script.read_text(encoding="utf-8")
|
|
)
|
|
|
|
setup_path = source_dir / "setup.py"
|
|
setup_path.write_text(
|
|
"from setuptools import find_packages, setup\n"
|
|
"setup(\n"
|
|
' name="behave-parallel",\n'
|
|
f' version="{BEHAVE_PARALLEL_VERSION}",\n'
|
|
" packages=find_packages(),\n"
|
|
' install_requires=["behave>=1.2.6"],\n'
|
|
" entry_points={\n"
|
|
' "console_scripts": '
|
|
'["behave-parallel=behave_parallel.cli:main"],\n'
|
|
" },\n"
|
|
")\n"
|
|
)
|
|
|
|
session.install("setuptools", "wheel")
|
|
session.install(str(source_dir))
|
|
|
|
|
|
# =============================================================================
|
|
|
|
|
|
# QUICK DEVELOPMENT SESSIONS (Fast - Run daily)
|
|
# =============================================================================
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def lint(session: nox.Session):
|
|
"""Check code formatting and linting."""
|
|
session.install("ruff>=0.15,<0.16", "semgrep>=1.45.0")
|
|
session.run(
|
|
"ruff",
|
|
"check",
|
|
"src/",
|
|
"scripts/",
|
|
"examples/",
|
|
"features/",
|
|
"robot/",
|
|
".opencode/",
|
|
)
|
|
# NOTE: Semgrep runs in audit mode (without --error) during the phased rollout.
|
|
# The codebase currently has ~337 existing broad-exception suppressions that must
|
|
# be triaged before enforcement mode is enabled. Once existing violations are
|
|
# annotated or fixed, change this to:
|
|
# session.run("semgrep", "--config=.semgrep.yml", "--error", "src/")
|
|
# See issue #9103 for the migration plan.
|
|
session.run("semgrep", "--config=.semgrep.yml", "src/")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def format(session: nox.Session):
|
|
"""Format code with ruff. Pass --check to verify without modifying."""
|
|
session.install("ruff>=0.15,<0.16")
|
|
session.run("ruff", "format", *session.posargs, ".")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def typecheck(session: nox.Session):
|
|
"""Check types with pyright."""
|
|
session.install("pyright")
|
|
session.install("-e", ".")
|
|
session.run("pyright", stderr=sys.stdout)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def unit_tests(session: nox.Session):
|
|
"""Run BDD tests with Behave."""
|
|
session.install("-e", ".[tests]")
|
|
# Explicitly ensure a2a-sdk is installed for A2A SDK dependency tests
|
|
session.install("a2a-sdk>=0.3.0")
|
|
_install_behave_parallel(session)
|
|
|
|
# Build a pre-migrated template DB so each scenario can copy it
|
|
# instead of running 25 Alembic migrations from scratch.
|
|
template_path = _create_template_db(session)
|
|
session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path
|
|
|
|
# Remove ANSI escape codes.
|
|
session.env["TERM"] = "dumb"
|
|
|
|
behave_cmd = session.bin + "/behave-parallel"
|
|
parallel_args = _behave_parallel_args(session.posargs)
|
|
|
|
# If specific feature files are passed, run only those files
|
|
has_feature_files = any(arg.endswith(".feature") for arg in session.posargs)
|
|
if has_feature_files:
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
*session.posargs,
|
|
]
|
|
else:
|
|
args = [
|
|
behave_cmd,
|
|
"-q",
|
|
*parallel_args,
|
|
"features/",
|
|
*session.posargs,
|
|
]
|
|
|
|
session.env["PYTHONPATH"] = str(Path("src").resolve())
|
|
# Prevent Rich from injecting ANSI escape codes into machine-readable
|
|
# CLI output (JSON/YAML) which causes json.loads() failures on CI.
|
|
session.env["NO_COLOR"] = "1"
|
|
# Pre-compile features/ bytecode so forked parallel workers don't race
|
|
# to write .pyc files simultaneously. Overlayfs copy-up locks cause
|
|
# open() to deadlock when N workers all compile uncached step files at
|
|
# the same time (thundering-herd on __pycache__).
|
|
session.run("python", "-m", "compileall", "-q", "features/")
|
|
session.run(*args)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def docs(session: nox.Session):
|
|
"""Build documentation with MkDocs."""
|
|
session.install("-e", ".[docs]")
|
|
# Suppress DeprecationWarning from the third-party `rx` package which calls
|
|
# datetime.datetime.utcfromtimestamp() (deprecated since Python 3.12).
|
|
session.env["PYTHONWARNINGS"] = "ignore::DeprecationWarning:rx"
|
|
session.run("mkdocs", "build")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def serve_docs(session: nox.Session):
|
|
"""Serve docs locally for development."""
|
|
session.install("-e", ".[docs]")
|
|
# Suppress DeprecationWarning from the third-party `rx` package which calls
|
|
# datetime.datetime.utcfromtimestamp() (deprecated since Python 3.12).
|
|
session.env["PYTHONWARNINGS"] = "ignore::DeprecationWarning:rx"
|
|
session.run("mkdocs", "serve", "--dev-addr", "0.0.0.0:8000")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def build(session: nox.Session):
|
|
"""Build the wheel distribution."""
|
|
session.install("build", "pip")
|
|
session.run("python", "-m", "build", "--wheel")
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def integration_tests(session: nox.Session):
|
|
"""Run Robot Framework integration tests (parallel via pabot).
|
|
|
|
Defaults to parallelism capped at 6 processes (min(6, cpus)) to avoid
|
|
resource pressure / LLM 429 flakes in CI while still using the runner.
|
|
Override via TEST_PROCESSES or by passing --processes/--processes=N in
|
|
session arguments.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
session.env["NO_COLOR"] = "1"
|
|
# Override PYTHONPATH so the editable install (src/) for this project
|
|
# is used instead of any inherited PYTHONPATH from the outer environment.
|
|
session.env["PYTHONPATH"] = "src"
|
|
|
|
# Propagate venv bin to PATH so Run Process in robot files finds
|
|
# the venv's python/robot rather than the system copies.
|
|
venv_bin = os.path.join(session.virtualenv.location, "bin")
|
|
session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
# and clear stale pabot worker artifacts from previous interrupted runs.
|
|
os.makedirs("build/reports/robot", exist_ok=True)
|
|
shutil.rmtree("build/reports/robot/pabot_results", ignore_errors=True)
|
|
|
|
# Pass the venv Python path explicitly so Run Process calls use it
|
|
# instead of relying on PATH (which may not propagate to subprocesses).
|
|
venv_python = os.path.join(venv_bin, "python")
|
|
|
|
# Pre-compile bytecode so that parallel pabot workers (and the
|
|
# Python sub-processes they spawn via ``Run Python Script``) can
|
|
# read cached .pyc files instead of each cold-compiling every
|
|
# module from source simultaneously — avoids a thundering-herd
|
|
# race on CI runners with high core counts.
|
|
session.run("python", "-m", "compileall", "-q", "src/")
|
|
|
|
# Build a pre-migrated template DB so helper scripts that call
|
|
# setup_workspace() can copy it instead of running 25+ Alembic
|
|
# migrations per test — critical for parallel pabot execution.
|
|
template_path = _create_template_db(session)
|
|
session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path
|
|
|
|
pabot_args, robot_args = _split_pabot_args(session.posargs)
|
|
parallel_args = _pabot_parallel_args(pabot_args)
|
|
|
|
# TDD expected-fail listener — inverts results for @tdd_expected_fail
|
|
# tagged tests and validates TDD tag combinations.
|
|
# See CONTRIBUTING.md > TDD Issue Test Tags.
|
|
# Resolved relative to this file (not CWD) so ``nox`` invocations from
|
|
# a non-root directory still find the listener module.
|
|
tdd_listener = str(
|
|
Path(__file__).parent / "robot" / "tdd_expected_fail_listener.py"
|
|
)
|
|
|
|
session.run(
|
|
"pabot",
|
|
*parallel_args,
|
|
*pabot_args,
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"--variable",
|
|
f"PYTHON:{venv_python}",
|
|
"--listener",
|
|
tdd_listener,
|
|
"--exclude",
|
|
"slow",
|
|
"--exclude",
|
|
"discovery",
|
|
"--exclude",
|
|
"code_blocks",
|
|
"--exclude",
|
|
"wip",
|
|
"--exclude",
|
|
"E2E",
|
|
"--exclude",
|
|
"tdd_fixture",
|
|
*robot_args,
|
|
"robot/",
|
|
)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def slow_integration_tests(session: nox.Session):
|
|
"""Run Robot Framework slow integration tests (parallel via pabot).
|
|
|
|
Runs all tests tagged ``slow`` that are excluded from the standard
|
|
``integration_tests`` session. Defaults to parallelism capped at 6
|
|
processes (min(6, cpus)) to avoid resource pressure / LLM 429 flakes in
|
|
CI. Override via TEST_PROCESSES or by passing --processes/--processes=N
|
|
in session arguments.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
session.env["NO_COLOR"] = "1"
|
|
# Override PYTHONPATH so the editable install (src/) for this project
|
|
# is used instead of any inherited PYTHONPATH from the outer environment.
|
|
session.env["PYTHONPATH"] = "src"
|
|
|
|
# Propagate venv bin to PATH so Run Process in robot files finds
|
|
# the venv's python/robot rather than the system copies.
|
|
venv_bin = os.path.join(session.virtualenv.location, "bin")
|
|
session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build/reports/robot", exist_ok=True)
|
|
|
|
# Pass the venv Python path explicitly so Run Process calls use it
|
|
# instead of relying on PATH (which may not propagate to subprocesses).
|
|
venv_python = os.path.join(venv_bin, "python")
|
|
|
|
# Pre-compile bytecode so that parallel pabot workers (and the
|
|
# Python sub-processes they spawn via ``Run Python Script``) can
|
|
# read cached .pyc files instead of each cold-compiling every
|
|
# module from source simultaneously — avoids a thundering-herd
|
|
# race on CI runners with high core counts.
|
|
session.run("python", "-m", "compileall", "-q", "src/")
|
|
|
|
# Build a pre-migrated template DB so helper scripts that call
|
|
# setup_workspace() can copy it instead of running 25+ Alembic
|
|
# migrations per test — critical for parallel pabot execution.
|
|
template_path = _create_template_db(session)
|
|
session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path
|
|
|
|
pabot_args, robot_args = _split_pabot_args(session.posargs)
|
|
parallel_args = _pabot_parallel_args(pabot_args)
|
|
|
|
# TDD expected-fail listener — inverts results for @tdd_expected_fail
|
|
# tagged tests and validates TDD tag combinations.
|
|
# See CONTRIBUTING.md > TDD Issue Test Tags.
|
|
# Resolved relative to this file (not CWD) so ``nox`` invocations from
|
|
# a non-root directory still find the listener module.
|
|
tdd_listener = str(
|
|
Path(__file__).parent / "robot" / "tdd_expected_fail_listener.py"
|
|
)
|
|
|
|
session.run(
|
|
"pabot",
|
|
*parallel_args,
|
|
*pabot_args,
|
|
"--outputdir",
|
|
"build/reports/robot",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"--variable",
|
|
f"PYTHON:{venv_python}",
|
|
"--listener",
|
|
tdd_listener,
|
|
"--include",
|
|
"slow",
|
|
"--exclude",
|
|
"discovery",
|
|
"--exclude",
|
|
"code_blocks",
|
|
"--exclude",
|
|
"wip",
|
|
"--exclude",
|
|
"E2E",
|
|
"--exclude",
|
|
"tdd_fixture",
|
|
*robot_args,
|
|
"robot/",
|
|
)
|
|
|
|
|
|
@nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv")
|
|
def e2e_tests(session: nox.Session):
|
|
"""Run end-to-end Robot Framework tests with real LLM API keys.
|
|
|
|
E2E tests use zero mocking — they exercise the real CleverAgents CLI
|
|
against real LLM API keys (Anthropic/OpenAI). Tests are tagged with
|
|
``E2E`` and live in the ``robot/e2e/`` directory.
|
|
|
|
This session is NOT included in the default ``nox`` run because it
|
|
requires real API keys. Run explicitly via ``nox -s e2e_tests``.
|
|
|
|
Tests that require LLM API keys will skip gracefully when the keys
|
|
are not present in the environment.
|
|
|
|
Parallelism is controlled by the ``TEST_PROCESSES`` environment variable
|
|
(default: min(cpu_count, 2)). Override via ``TEST_PROCESSES=N nox -s
|
|
e2e_tests`` or by passing ``--processes N`` as a session argument.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
session.env["CLEVERAGENTS_AUTO_APPLY_MIGRATIONS"] = "true"
|
|
session.env["NO_COLOR"] = "1"
|
|
session.env["PYTHONPATH"] = "src"
|
|
|
|
# Propagate venv bin to PATH so Run Process in robot files finds
|
|
# the venv's python/robot rather than the system copies.
|
|
venv_bin = os.path.join(session.virtualenv.location, "bin")
|
|
session.env["PATH"] = venv_bin + os.pathsep + os.environ.get("PATH", "")
|
|
venv_python = os.path.join(venv_bin, "python")
|
|
|
|
# Ensure output directory exists
|
|
os.makedirs("build/reports/robot-e2e", exist_ok=True)
|
|
|
|
# Pre-compile bytecode so that parallel pabot workers (and the
|
|
# Python sub-processes they spawn via ``Run Process``) can read
|
|
# cached .pyc files instead of each cold-compiling every module
|
|
# from source simultaneously — avoids a thundering-herd race on
|
|
# CI runners with high core counts.
|
|
session.run("python", "-m", "compileall", "-q", "src/")
|
|
|
|
# Build a pre-migrated template DB so helper scripts that call
|
|
# setup_workspace() can copy it instead of running 25+ Alembic
|
|
# migrations per test — reduces per-test setup from ~1-3 s to ~1 ms.
|
|
# Critical for parallel pabot execution where many suites start
|
|
# simultaneously and would otherwise all race to run migrations.
|
|
template_path = _create_template_db(session)
|
|
session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path
|
|
|
|
# Propagate LLM API keys from the environment into the session
|
|
# so that real E2E tests can authenticate with providers.
|
|
for key in (
|
|
"ANTHROPIC_API_KEY",
|
|
"OPENAI_API_KEY",
|
|
"GOOGLE_API_KEY",
|
|
"GEMINI_API_KEY",
|
|
):
|
|
value = os.environ.get(key)
|
|
if value:
|
|
session.env[key] = value
|
|
|
|
# Split posargs into pabot-specific args (--processes) and robot args.
|
|
pabot_args, robot_args = _split_pabot_args(session.posargs)
|
|
parallel_args = _pabot_parallel_args(pabot_args)
|
|
|
|
# TDD expected-fail listener — inverts results for @tdd_expected_fail
|
|
# tagged tests and validates TDD tag combinations.
|
|
# See CONTRIBUTING.md > TDD Issue Test Tags.
|
|
tdd_listener = str(
|
|
Path(__file__).parent / "robot" / "tdd_expected_fail_listener.py"
|
|
)
|
|
|
|
# Use pabot (parallel Robot Framework runner) instead of the sequential
|
|
# ``robot`` runner. Each E2E suite file runs in its own pabot worker
|
|
# process, so independent suites execute concurrently rather than
|
|
# serially. Suite-level isolation (separate CLEVERAGENTS_HOME per
|
|
# suite) ensures workers do not share database state.
|
|
session.run(
|
|
"pabot",
|
|
*parallel_args,
|
|
*pabot_args,
|
|
"--outputdir",
|
|
"build/reports/robot-e2e",
|
|
"--loglevel",
|
|
"INFO",
|
|
"--report",
|
|
"report.html",
|
|
"--log",
|
|
"log.html",
|
|
"--xunit",
|
|
"xunit.xml",
|
|
"--variable",
|
|
f"PYTHON:{venv_python}",
|
|
"--include",
|
|
"E2E",
|
|
"--listener",
|
|
tdd_listener,
|
|
*robot_args,
|
|
"robot/e2e/",
|
|
)
|
|
|
|
|
|
def _read_coverage_fail_under() -> float:
|
|
"""Single source of truth for the coverage floor (plan decision 2a):
|
|
``pyproject.toml [tool.coverage.report].fail_under``. slipcover does not
|
|
auto-read pyproject, so this session passes ``--fail-under`` explicitly
|
|
and reports this value in the summary line. Falls back to 96.5 (the
|
|
canonical floor, temporarily lowered for the @tdd_expected_fail backlog --
|
|
issues #4183/#4184) if the key is missing/unreadable.
|
|
|
|
Ratchet (decision 2b): the objective is 97% -- raise ``fail_under`` only
|
|
after observed master coverage has held >= target+buffer for N green
|
|
commits. The 100%-patch diff-gate keeps total coverage monotonic, so the
|
|
floor trails real coverage upward; it is not a precondition for shipping.
|
|
"""
|
|
try:
|
|
data = tomllib.loads((Path(__file__).parent / "pyproject.toml").read_text())
|
|
return float(data["tool"]["coverage"]["report"]["fail_under"])
|
|
except (OSError, KeyError, TypeError, ValueError, tomllib.TOMLDecodeError):
|
|
return 96.5
|
|
|
|
|
|
COVERAGE_THRESHOLD = _read_coverage_fail_under()
|
|
|
|
# Engine tunables (WS1). ``K`` bounds concurrent slipcover processes; ``N`` is
|
|
# the number of bin-packed chunks, decoupled from K (default ~3xK). ``N`` sets
|
|
# per-chunk peak RSS; ``K`` sets the concurrency multiplier (ceiling ~ K x peak).
|
|
COVERAGE_PROCESSES_DEFAULT = 4
|
|
|
|
|
|
_COUNT_SCENARIOS_SCRIPT = """
|
|
import json, sys
|
|
from behave.parser import parse_file
|
|
out = {}
|
|
for fp in sys.argv[2:]:
|
|
try:
|
|
f = parse_file(fp)
|
|
out[fp] = 1 if f is None else max(1, sum(1 for _ in f.walk_scenarios()))
|
|
except Exception:
|
|
out[fp] = 1
|
|
with open(sys.argv[1], "w") as fh:
|
|
json.dump(out, fh)
|
|
"""
|
|
|
|
|
|
def _count_scenarios_per_feature(
|
|
session: nox.Session, feature_paths: list[str]
|
|
) -> dict[str, int]:
|
|
"""Map each ``.feature`` path to its scenario count (outline-expanded).
|
|
|
|
Parses each feature with behave's gherkin parser and walks its scenarios
|
|
(so a ``Scenario Outline`` contributes one unit per ``Examples`` row, the
|
|
same unit the runner executes). Used as the bin-packing weight so chunks
|
|
balance by real work, not file count (naive contiguous split measured a
|
|
1.94 max/mean imbalance -- see plan V10).
|
|
|
|
behave lives in the *session* venv, not the nox orchestrator process, so
|
|
the parse runs via the session python in a subprocess writing JSON to a
|
|
file (keeps stdout clean). On any failure every file is weighted 1 so none
|
|
is silently dropped (bin-packing still runs, just by file count).
|
|
"""
|
|
counts_path = str(Path("build/scenario_counts.json").resolve())
|
|
proc = subprocess.run(
|
|
[
|
|
session.bin + "/python",
|
|
"-c",
|
|
_COUNT_SCENARIOS_SCRIPT,
|
|
counts_path,
|
|
*feature_paths,
|
|
],
|
|
env=_child_env(session),
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
session.log(
|
|
"scenario count failed; falling back to file-count bins: "
|
|
f"{proc.stderr.strip()[-300:]}"
|
|
)
|
|
return {fp: 1 for fp in feature_paths}
|
|
with open(counts_path) as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
def _bin_pack(
|
|
feature_paths: list[str], counts: dict[str, int], n_bins: int
|
|
) -> list[list[str]]:
|
|
"""Greedy largest-first bin-packing into ``n_bins`` least-loaded bins.
|
|
|
|
Sorts features by scenario count descending, then drops each into the
|
|
currently-lightest bin. Returns only non-empty bins (so the chunk count
|
|
never exceeds the feature count). This keeps per-chunk scenario load
|
|
balanced, bounding per-process peak RSS and tail wall-clock.
|
|
"""
|
|
n_bins = max(1, min(n_bins, len(feature_paths)))
|
|
ordered = sorted(feature_paths, key=lambda p: counts.get(p, 1), reverse=True)
|
|
bins: list[list[str]] = [[] for _ in range(n_bins)]
|
|
loads = [0] * n_bins
|
|
for fp in ordered:
|
|
i = loads.index(min(loads))
|
|
bins[i].append(fp)
|
|
loads[i] += counts.get(fp, 1)
|
|
return [b for b in bins if b]
|
|
|
|
|
|
def _child_env(session: nox.Session) -> dict[str, str]:
|
|
"""Environment for a slipcover child process.
|
|
|
|
Inherits the session env (PYTHONPATH, NO_COLOR, BEHAVE_PARALLEL_COVERAGE,
|
|
CLEVERAGENTS_TEMPLATE_DB) but DEFENSIVELY strips any inherited
|
|
``CLEVERAGENTS_DATABASE_URL``: ``before_all`` mktemps a per-process DB
|
|
only-if-unset, so a fixed value would make all K children share one DB and
|
|
collide (plan V15). Only the template DB is passed through.
|
|
"""
|
|
merged = {**os.environ, **session.env}
|
|
merged.pop("CLEVERAGENTS_DATABASE_URL", None)
|
|
# session.env may carry None-valued keys (nox unset markers); subprocess
|
|
# rejects non-str env values, so drop them.
|
|
env = {k: str(v) for k, v in merged.items() if v is not None}
|
|
return env
|
|
|
|
|
|
_MAXRSS_RE = re.compile(r"Maximum resident set size \(kbytes\):\s*(\d+)")
|
|
|
|
|
|
def _run_coverage_chunk(
|
|
venv_python: str,
|
|
source_paths: str,
|
|
omit_patterns: str,
|
|
behave_cmd: str,
|
|
chunk: list[str],
|
|
index: int,
|
|
env: dict[str, str],
|
|
) -> dict:
|
|
"""Run one slipcover subprocess over ``chunk`` and report its result.
|
|
|
|
Wraps the child in ``/usr/bin/time -v`` (when available) to capture peak
|
|
RSS. Child stdout+stderr are tee'd to ``build/coverage.<i>.log`` so a
|
|
failing chunk's diagnostics survive without buffering the whole suite's
|
|
output in memory. Returns a result dict for the caller to validate.
|
|
"""
|
|
out_json = f"build/coverage.{index}.json"
|
|
log_path = f"build/coverage.{index}.log"
|
|
Path(out_json).unlink(missing_ok=True)
|
|
|
|
slip_cmd = [
|
|
venv_python,
|
|
"-m",
|
|
"slipcover",
|
|
"--json",
|
|
"--out",
|
|
out_json,
|
|
"--source",
|
|
source_paths,
|
|
"--omit",
|
|
omit_patterns,
|
|
"--",
|
|
behave_cmd,
|
|
"-q",
|
|
"--no-capture",
|
|
*chunk,
|
|
]
|
|
time_bin = "/usr/bin/time"
|
|
cmd = [time_bin, "-v", *slip_cmd] if os.path.exists(time_bin) else slip_cmd
|
|
|
|
start = time.monotonic()
|
|
with open(log_path, "wb") as log_file:
|
|
proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, env=env)
|
|
returncode = proc.wait()
|
|
wall = time.monotonic() - start
|
|
|
|
try:
|
|
log_text = Path(log_path).read_text(errors="replace")
|
|
except OSError:
|
|
log_text = ""
|
|
rss_match = _MAXRSS_RE.search(log_text)
|
|
peak_rss_kb = int(rss_match.group(1)) if rss_match else None
|
|
|
|
return {
|
|
"index": index,
|
|
"returncode": returncode,
|
|
"out_json": out_json,
|
|
"log_path": log_path,
|
|
"wall": wall,
|
|
"peak_rss_kb": peak_rss_kb,
|
|
"n_features": len(chunk),
|
|
}
|
|
|
|
|
|
def _validate_coverage_chunk(result: dict) -> str | None:
|
|
"""Three-part liveness check for a chunk. Returns a diagnostic or None.
|
|
|
|
A chunk is alive iff: exit code in {0, 1} (0 = pass, 1 = test failures,
|
|
both still emit coverage) AND its JSON parses AND it recorded a non-empty
|
|
executed-line set. Anything else (OOM/137, crash, truncated JSON, empty
|
|
data) is a DEAD chunk -- the caller must fail the whole job rather than
|
|
merge survivors, which would report false-low coverage.
|
|
"""
|
|
idx = result["index"]
|
|
rc = result["returncode"]
|
|
if rc not in (0, 1):
|
|
return f"chunk {idx} exited {rc} (e.g. 137 = OOM/kill)"
|
|
|
|
out_json = result["out_json"]
|
|
if not os.path.exists(out_json):
|
|
return f"chunk {idx} produced no JSON ({out_json} missing)"
|
|
try:
|
|
with open(out_json) as f:
|
|
payload = json.load(f)
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
return f"chunk {idx} JSON did not parse: {exc}"
|
|
|
|
files = payload.get("files") or {}
|
|
executed_total = sum(
|
|
len(meta.get("executed_lines") or []) for meta in files.values()
|
|
)
|
|
if executed_total == 0:
|
|
return f"chunk {idx} recorded an empty executed-line set"
|
|
return None
|
|
|
|
|
|
def _log_tail(log_path: str, max_lines: int = 40) -> str:
|
|
try:
|
|
lines = Path(log_path).read_text(errors="replace").splitlines()
|
|
except OSError:
|
|
return "(log unavailable)"
|
|
return "\n".join(lines[-max_lines:])
|
|
|
|
|
|
def _run_parallel_coverage(
|
|
session: nox.Session,
|
|
behave_cmd: str,
|
|
source_paths: str,
|
|
omit_patterns: str,
|
|
) -> None:
|
|
"""Full-suite coverage via K concurrent slipcover processes over N chunks.
|
|
|
|
Enumerates ``features/``, bin-packs by scenario count into N chunks, runs
|
|
them K-at-a-time, fails loud on any dead chunk, and merges the survivors'
|
|
JSON into ``build/coverage.json``. The downstream xml/report/summary steps
|
|
consume the merged JSON unchanged.
|
|
"""
|
|
# Enumerate features directly rather than importing run_behave_parallel:
|
|
# that module's top level imports behave/behave_parallel, which live only in
|
|
# the *session* venv, not the nox orchestrator process -- importing it here
|
|
# would raise ModuleNotFoundError. This glob mirrors the runner's
|
|
# ``_iter_features`` directory branch and keeps the engine self-contained.
|
|
feature_paths = sorted(str(fp) for fp in Path("features").rglob("*.feature"))
|
|
if not feature_paths:
|
|
session.error("COVERAGE FAILED: no feature files found under features/")
|
|
|
|
k = max(1, int(os.environ.get("COVERAGE_PROCESSES", COVERAGE_PROCESSES_DEFAULT)))
|
|
n = int(os.environ.get("COVERAGE_CHUNKS", 3 * k))
|
|
n = max(k, min(n, len(feature_paths)))
|
|
|
|
counts = _count_scenarios_per_feature(session, feature_paths)
|
|
total_scenarios = sum(counts.values())
|
|
chunks = _bin_pack(feature_paths, counts, n)
|
|
chunk_loads = [sum(counts.get(fp, 1) for fp in c) for c in chunks]
|
|
session.log(
|
|
f"coverage engine: K={k} processes, N={len(chunks)} chunks, "
|
|
f"{len(feature_paths)} features / {total_scenarios} scenarios; "
|
|
f"chunk loads min/max={min(chunk_loads)}/{max(chunk_loads)}"
|
|
)
|
|
|
|
venv_python = session.bin + "/python"
|
|
env = _child_env(session)
|
|
|
|
suite_start = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=k) as pool:
|
|
results = list(
|
|
pool.map(
|
|
lambda item: _run_coverage_chunk(
|
|
venv_python,
|
|
source_paths,
|
|
omit_patterns,
|
|
behave_cmd,
|
|
item[1],
|
|
item[0],
|
|
env,
|
|
),
|
|
enumerate(chunks),
|
|
)
|
|
)
|
|
suite_wall = time.monotonic() - suite_start
|
|
|
|
# Per-chunk telemetry (peak RSS + wall) for the equivalence report.
|
|
for r in sorted(results, key=lambda r: r["index"]):
|
|
rss = f"{r['peak_rss_kb'] / 1024:.0f}MB" if r["peak_rss_kb"] else "n/a"
|
|
session.log(
|
|
f" chunk {r['index']}: rc={r['returncode']} "
|
|
f"features={r['n_features']} wall={r['wall']:.0f}s peakRSS={rss}"
|
|
)
|
|
peaks = [r["peak_rss_kb"] for r in results if r["peak_rss_kb"]]
|
|
if peaks:
|
|
session.log(
|
|
f"coverage engine wall={suite_wall:.0f}s "
|
|
f"max per-chunk peakRSS={max(peaks) / 1024:.0f}MB "
|
|
f"(ceiling ~ K x peak = {k * max(peaks) / 1024:.0f}MB)"
|
|
)
|
|
|
|
# Fail loud on ANY dead chunk -- never merge survivors (a partial merge
|
|
# reports artificially-low coverage = the exact false-failure we kill).
|
|
diagnostics = [d for d in (_validate_coverage_chunk(r) for r in results) if d]
|
|
if diagnostics:
|
|
detail = "\n".join(diagnostics)
|
|
worst = next(r for r in results if _validate_coverage_chunk(r) is not None)
|
|
session.error(
|
|
"COVERAGE FAILED: dead chunk(s) -- refusing to merge survivors "
|
|
f"(would report false-low coverage).\n{detail}\n"
|
|
f"--- tail of {worst['log_path']} ---\n{_log_tail(worst['log_path'])}"
|
|
)
|
|
|
|
# Merge the per-chunk JSON into the canonical coverage.json (verified
|
|
# correct union -- plan V1). The downstream xml/report steps run on this.
|
|
merge_inputs = [r["out_json"] for r in sorted(results, key=lambda r: r["index"])]
|
|
session.run(
|
|
"python",
|
|
"-m",
|
|
"slipcover",
|
|
"--merge",
|
|
*merge_inputs,
|
|
"--json",
|
|
"--out",
|
|
"build/coverage.json",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def coverage_report(session: nox.Session):
|
|
"""Generate coverage report from Behave tests.
|
|
|
|
Full-suite runs use a K-way parallel engine: ``features/`` is bin-packed
|
|
by scenario count into N chunks, each measured by its own short-lived
|
|
``slipcover --json`` process (K concurrent), then the per-chunk JSON is
|
|
merged. This bounds per-process peak RSS (killing the single-process
|
|
OOM/reaper collapse) and cuts wall-clock, while remaining bit-for-bit
|
|
equivalent to the single-process measurement. K = env ``COVERAGE_PROCESSES``
|
|
(default 4); N = env ``COVERAGE_CHUNKS`` (default ~3xK).
|
|
|
|
A *targeted* run (posargs naming ``.feature``) keeps the single-process
|
|
path, preserving the scenario-specifier recipe.
|
|
|
|
Coverage threshold is enforced at >= the pyproject
|
|
``[tool.coverage.report].fail_under`` floor (plan decision 2a).
|
|
|
|
On success, emits: COVERAGE OK: <pct>% (threshold: <floor>%)
|
|
On failure, emits: COVERAGE FAILED: <pct>% < <floor>% threshold
|
|
Both are single-line, CI-parseable summary strings.
|
|
"""
|
|
session.install("-e", ".[tests]")
|
|
_install_behave_parallel(session)
|
|
|
|
os.makedirs("build", exist_ok=True)
|
|
|
|
# Build a pre-migrated template DB so each scenario can copy it
|
|
# instead of running 25 Alembic migrations from scratch.
|
|
template_path = _create_template_db(session)
|
|
session.env["CLEVERAGENTS_TEMPLATE_DB"] = template_path
|
|
|
|
session.env["PYTHONPATH"] = str(Path("src").resolve())
|
|
session.env["NO_COLOR"] = "1"
|
|
source_paths = "src"
|
|
omit_patterns = ",".join(
|
|
[
|
|
"*/tests/*",
|
|
"*/test_*",
|
|
"features/*",
|
|
"*/features/*",
|
|
"*/__pycache__/*",
|
|
"*/site-packages/*",
|
|
"*/dependency_injector/*",
|
|
"*/venv/*",
|
|
"*/.venv/*",
|
|
"*/.nox/*",
|
|
"src/cleveragents/discovery/*",
|
|
"src/cleveragents/tui/materializer.py",
|
|
# TYPE_CHECKING-only re-export shim: 100% of its "missing" lines are
|
|
# inside an `if TYPE_CHECKING:` block (never executes at runtime),
|
|
# and slipcover has no per-line pragma to exclude them. Omit the
|
|
# whole module so those uncoverable lines stop dragging the total.
|
|
"src/cleveragents/application/services/__init__.py",
|
|
]
|
|
)
|
|
|
|
# Force sequential mode so slipcover wraps the entire process.
|
|
session.env["BEHAVE_PARALLEL_COVERAGE"] = "1"
|
|
|
|
# Clean up any existing slipcover / per-chunk data
|
|
for path in Path("build").glob(".slipcover.*.json"):
|
|
path.unlink()
|
|
for path in Path("build").glob("coverage.*.json"):
|
|
path.unlink()
|
|
|
|
behave_cmd = session.bin + "/behave-parallel"
|
|
has_feature_files = any(arg.endswith(".feature") for arg in session.posargs)
|
|
if has_feature_files:
|
|
# Targeted run: a small, scoped selection. Keep the single-process
|
|
# path -- chunking adds startup overhead for no balance benefit. Allow
|
|
# exit code 1 (test failures) -- coverage data is still produced.
|
|
session.run(
|
|
"python",
|
|
"-m",
|
|
"slipcover",
|
|
"--json",
|
|
"--out",
|
|
"build/coverage.json",
|
|
"--source",
|
|
source_paths,
|
|
"--omit",
|
|
omit_patterns,
|
|
"--",
|
|
behave_cmd,
|
|
"-q",
|
|
"--no-capture",
|
|
*session.posargs,
|
|
success_codes=[0, 1],
|
|
)
|
|
else:
|
|
# Full suite: K-way parallel fan-out -> merged build/coverage.json.
|
|
_run_parallel_coverage(session, behave_cmd, source_paths, omit_patterns)
|
|
|
|
# Generate XML report for CI
|
|
session.run(
|
|
"python",
|
|
"-m",
|
|
"slipcover",
|
|
"--merge",
|
|
"build/coverage.json",
|
|
"--xml",
|
|
"--out",
|
|
"build/coverage.xml",
|
|
)
|
|
|
|
# Generate terminal report and check threshold.
|
|
# slipcover exits with code 2 if below threshold; nox intercepts the exit.
|
|
report_path = "build/coverage-report.txt"
|
|
session.run(
|
|
"python",
|
|
"-m",
|
|
"slipcover",
|
|
"--merge",
|
|
"build/coverage.json",
|
|
"--out",
|
|
report_path,
|
|
"--source",
|
|
source_paths,
|
|
"--omit",
|
|
omit_patterns,
|
|
f"--fail-under={COVERAGE_THRESHOLD}",
|
|
success_codes=[0, 2],
|
|
silent=False,
|
|
)
|
|
|
|
try:
|
|
with open(report_path) as report_file:
|
|
print(report_file.read(), end="")
|
|
except FileNotFoundError:
|
|
session.log(f"Slipcover report not found at {report_path}")
|
|
|
|
# Parse the total percentage from the slipcover JSON report for the summary
|
|
total_pct: float = 0.0
|
|
try:
|
|
with open("build/coverage.json") as f:
|
|
payload = json.load(f)
|
|
|
|
summary = payload.get("summary") or payload.get("totals") or {}
|
|
percent_value = None
|
|
for key in (
|
|
"percent_covered",
|
|
"percent_covered_display",
|
|
"line_coverage_percent",
|
|
):
|
|
percent_value = summary.get(key)
|
|
if percent_value is not None:
|
|
break
|
|
|
|
if percent_value is None:
|
|
covered = (
|
|
summary.get("lines_covered")
|
|
or summary.get("covered_lines")
|
|
or summary.get("covered")
|
|
)
|
|
total = (
|
|
summary.get("lines")
|
|
or summary.get("total_lines")
|
|
or summary.get("num_lines")
|
|
or summary.get("total")
|
|
)
|
|
if covered is not None and total:
|
|
percent_value = (float(covered) / float(total)) * 100.0
|
|
|
|
if isinstance(percent_value, str):
|
|
percent_value = percent_value.strip().rstrip("%")
|
|
|
|
if percent_value is not None:
|
|
total_pct = float(percent_value)
|
|
except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError):
|
|
total_pct = 0.0
|
|
|
|
rounded_pct = round(total_pct, 1)
|
|
|
|
if rounded_pct >= COVERAGE_THRESHOLD:
|
|
session.log(f"COVERAGE OK: {rounded_pct}% (threshold: {COVERAGE_THRESHOLD:g}%)")
|
|
else:
|
|
session.error(
|
|
f"COVERAGE FAILED: {rounded_pct}% < {COVERAGE_THRESHOLD:g}% threshold"
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def pre_commit(session: nox.Session):
|
|
"""Run all pre-commit hooks on all files."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("pre-commit", "run", "--all-files", *session.posargs)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def security_scan(session: nox.Session):
|
|
"""Run security checks matching Forgejo CI: bandit + semgrep + vulture.
|
|
|
|
Steps:
|
|
1. Bandit medium-severity report (non-blocking JSON export to build/)
|
|
2. Bandit high-severity gate (blocks on findings)
|
|
3. Semgrep custom rules from .semgrep.yml (blocks on ERROR-level findings)
|
|
4. Vulture dead-code detection (blocks on confidence >=80%)
|
|
|
|
Severity gates:
|
|
- Bandit HIGH: hard fail (blocks merge)
|
|
- Bandit MEDIUM: report-only (review, do not block)
|
|
- Semgrep ERROR: hard fail (eval/exec/os.system detection)
|
|
- Semgrep WARNING: report-only (pickle.loads)
|
|
- Vulture >=80% confidence: hard fail
|
|
"""
|
|
session.install("-e", ".[dev]")
|
|
# Python 3.13 no longer bundles pkg_resources. Semgrep transitively imports
|
|
# it via opentelemetry instrumentation, so pin setuptools to a version that
|
|
# still provides pkg_resources.
|
|
session.install("setuptools<81")
|
|
|
|
# Ensure output directory exists (CI starts with a clean checkout)
|
|
os.makedirs("build", exist_ok=True)
|
|
|
|
# Step 1: Bandit - medium-severity report (non-blocking, mirrors CI JSON export)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"medium",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
"build/bandit-report.json",
|
|
success_codes=[0, 1],
|
|
)
|
|
|
|
# Step 2: Bandit - high-severity gate (blocks on findings)
|
|
session.run(
|
|
"bandit",
|
|
"-c",
|
|
"pyproject.toml",
|
|
"-r",
|
|
"src/cleveragents",
|
|
"--severity-level",
|
|
"high",
|
|
)
|
|
|
|
# Step 3: Semgrep custom rules (blocks on ERROR-severity matches)
|
|
semgrep_config = Path(".semgrep.yml")
|
|
if semgrep_config.exists():
|
|
session.run(
|
|
"semgrep",
|
|
"--config=.semgrep.yml",
|
|
"--error",
|
|
"--quiet",
|
|
"src/",
|
|
success_codes=[0, 1],
|
|
)
|
|
else:
|
|
session.warn("No .semgrep.yml found, skipping semgrep scan")
|
|
|
|
# Step 4: Vulture dead-code detection
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def dead_code(session: nox.Session):
|
|
"""Run vulture to detect dead code."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"vulture",
|
|
"src/cleveragents",
|
|
"vulture_whitelist.py",
|
|
"--min-confidence",
|
|
"80",
|
|
"--exclude",
|
|
"src/cleveragents/discovery",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def complexity(session: nox.Session):
|
|
"""Check code complexity using radon."""
|
|
session.install("-e", ".[dev]")
|
|
session.run(
|
|
"radon",
|
|
"cc",
|
|
"src/cleveragents",
|
|
"--min",
|
|
"C",
|
|
"--show-complexity",
|
|
"--total-average",
|
|
)
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def adr_compliance(session: nox.Session):
|
|
"""Check code compliance with Architecture Decision Records."""
|
|
session.install("-e", ".[dev]")
|
|
session.run("python", "scripts/check-adr-compliance.py")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def benchmark(session: nox.Session):
|
|
"""Run Airspeed Velocity benchmarks and publish results."""
|
|
session.install("-e", ".[tests]")
|
|
config_path = "asv.conf.json"
|
|
session.run(
|
|
"asv",
|
|
"machine",
|
|
"--machine=forgejo-runner",
|
|
"--os=Linux_6.x",
|
|
"--arch=x86_64",
|
|
"--num_cpu=32",
|
|
"--ram=32GB",
|
|
"--cpu=AMD",
|
|
f"--config={config_path}",
|
|
)
|
|
session.run(
|
|
"asv",
|
|
"run",
|
|
"--machine=forgejo-runner",
|
|
"--append-samples",
|
|
"--launch-method=spawn",
|
|
"--show-stderr",
|
|
"--verbose",
|
|
f"--config={config_path}",
|
|
success_codes=[0, 2],
|
|
)
|
|
session.run("asv", "publish", f"--config={config_path}")
|
|
|
|
|
|
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
|
def benchmark_regression(session: nox.Session):
|
|
"""Run Airspeed Velocity benchmarks regression test."""
|
|
session.install("-e", ".[tests]")
|
|
config_path = "asv.conf.json"
|
|
asv_base_sha = os.environ.get("ASV_BASE_SHA", "master")
|
|
session.run(
|
|
"asv",
|
|
"machine",
|
|
"--machine=forgejo-runner",
|
|
"--os=Linux_6.x",
|
|
"--arch=x86_64",
|
|
"--num_cpu=32",
|
|
"--ram=32GB",
|
|
"--cpu=AMD",
|
|
f"--config={config_path}",
|
|
)
|
|
session.run(
|
|
"asv",
|
|
"continuous",
|
|
"--machine=forgejo-runner",
|
|
"--append-samples",
|
|
"--show-stderr",
|
|
"--verbose",
|
|
"--factor=1.50",
|
|
f"--config={config_path}",
|
|
asv_base_sha,
|
|
"HEAD",
|
|
success_codes=[0, 1, 2],
|
|
)
|
|
session.run("asv", "publish", f"--config={config_path}")
|
|
|
|
|
|
# Sessions to run by default when running `nox` without arguments
|
|
nox.options.sessions = [
|
|
"lint", # ~5-10 seconds
|
|
"format", # ~5-10 seconds (auto-fixes issues)
|
|
"typecheck", # ~10-20 seconds
|
|
"security_scan", # bandit + vulture (mirrors Forgejo CI security job)
|
|
"dead_code", # vulture dead-code detection
|
|
"unit_tests", # ~30-60 seconds (without ML deps)
|
|
"integration_tests",
|
|
"docs", # ~10-30 seconds
|
|
"build", # ~5-10 seconds
|
|
"benchmark", # ASV benchmarks for performance tracking
|
|
"coverage_report", # ~30-60 seconds (without ML deps)
|
|
]
|