CoreRasurae
455 lines
17 KiB
Python
455 lines
17 KiB
Python
"""Step definitions for architecture validation."""
|
|
|
|
import ast
|
|
import importlib
|
|
import pkgutil
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from behave import given, then, when
|
|
|
|
|
|
@given('the ADR directory exists at "{path}"')
|
|
def step_adr_directory_exists(context, path):
|
|
"""Check if the ADR directory exists."""
|
|
context.adr_dir = Path(path)
|
|
assert context.adr_dir.exists(), f"ADR directory {path} does not exist"
|
|
assert context.adr_dir.is_dir(), f"{path} is not a directory"
|
|
|
|
|
|
@given('the ADR directory "{path}" may be absent')
|
|
def step_adr_directory_may_be_absent(context, path):
|
|
"""Record the ADR directory path; it may not exist."""
|
|
context.adr_dir = Path(path)
|
|
context.adr_dir_exists = context.adr_dir.exists() and context.adr_dir.is_dir()
|
|
|
|
|
|
@then("the ADR check should be skipped when the directory is missing")
|
|
def step_adr_skip_when_missing(context):
|
|
"""Pass when the ADR directory is absent (docs removed from repo)."""
|
|
if context.adr_dir_exists:
|
|
# If someone recreates the directory, this test still passes
|
|
adr_files = list(context.adr_dir.glob("ADR-*.md"))
|
|
assert len(adr_files) >= 0 # Just verify glob works
|
|
# When directory is missing, this step passes (expected state)
|
|
|
|
|
|
@when("I check for ADR files")
|
|
def step_check_adr_files(context):
|
|
"""Find all ADR files in the directory."""
|
|
context.adr_files = list(context.adr_dir.glob("ADR-*.md"))
|
|
|
|
|
|
@then("I should find at least {count:d} ADR files")
|
|
def step_verify_adr_count(context, count):
|
|
"""Verify minimum number of ADR files."""
|
|
assert len(context.adr_files) >= count, (
|
|
f"Expected at least {count} ADR files, found {len(context.adr_files)}"
|
|
)
|
|
|
|
|
|
@then("each ADR should have a valid format with status")
|
|
def step_verify_adr_format(context):
|
|
"""Verify each ADR follows the expected format."""
|
|
for adr_file in context.adr_files:
|
|
content = adr_file.read_text()
|
|
# Check for required sections
|
|
assert "## Status" in content, f"{adr_file.name} missing Status section"
|
|
assert "## Context" in content, f"{adr_file.name} missing Context section"
|
|
assert "## Decision" in content, f"{adr_file.name} missing Decision section"
|
|
assert "## Consequences" in content, (
|
|
f"{adr_file.name} missing Consequences section"
|
|
)
|
|
|
|
|
|
@given('the source directory exists at "{path}"')
|
|
def step_source_directory_exists(context, path):
|
|
"""Check if the source directory exists."""
|
|
context.src_dir = Path(path)
|
|
assert context.src_dir.exists(), f"Source directory {path} does not exist"
|
|
|
|
|
|
@when("I check the package structure")
|
|
def step_check_package_structure(context):
|
|
"""List all packages in the source directory."""
|
|
try:
|
|
# Ensure the directory exists
|
|
if not context.src_dir.exists():
|
|
raise AssertionError(f"Source directory {context.src_dir} does not exist")
|
|
|
|
# List all items in the directory with better error handling
|
|
context.packages = {}
|
|
for item in context.src_dir.iterdir():
|
|
try:
|
|
if item.is_dir() and not item.name.startswith("__"):
|
|
# Store all packages including discovery
|
|
# Handle expected vs actual in verification
|
|
context.packages[item.name] = item
|
|
except (OSError, PermissionError) as e:
|
|
# Log the error but continue processing other items
|
|
print(f"Warning: Could not check {item}: {e}")
|
|
continue
|
|
|
|
except Exception as e:
|
|
raise AssertionError(f"Failed to check package structure: {e}") from e
|
|
|
|
|
|
@then("I should find these main packages:")
|
|
def step_verify_packages(context):
|
|
"""Verify expected packages exist and are proper Python packages."""
|
|
# Check if packages were properly collected
|
|
if not hasattr(context, "packages"):
|
|
raise AssertionError(
|
|
"Package structure was not checked properly - 'packages' attribute missing"
|
|
)
|
|
|
|
# Parse the table to get expected packages and their descriptions
|
|
expected_packages = {
|
|
row["package"]: row.get("description", "") for row in context.table
|
|
}
|
|
actual_packages = set(context.packages.keys())
|
|
|
|
# Check for missing packages
|
|
missing = set(expected_packages.keys()) - actual_packages
|
|
if missing:
|
|
extra = actual_packages - set(expected_packages.keys())
|
|
error_msg = f"Missing packages: {missing}"
|
|
if extra:
|
|
error_msg += f"\nUnexpected packages found: {extra}"
|
|
error_msg += f"\nActual packages: {sorted(actual_packages)}"
|
|
error_msg += f"\nExpected packages: {sorted(expected_packages.keys())}"
|
|
raise AssertionError(error_msg)
|
|
|
|
# Verify each expected package is a proper Python package
|
|
for package_name in expected_packages:
|
|
package_path = context.packages[package_name]
|
|
init_file = package_path / "__init__.py"
|
|
if not init_file.exists():
|
|
raise AssertionError(
|
|
f"Package '{package_name}' at {package_path} is not a "
|
|
f"proper Python package (__init__.py not found)"
|
|
)
|
|
|
|
|
|
@given("the package structure is defined")
|
|
def step_package_structure_defined(context):
|
|
"""Set up package structure for import analysis."""
|
|
context.src_dir = Path("src/cleveragents")
|
|
context.packages = {
|
|
p.name: p
|
|
for p in context.src_dir.iterdir()
|
|
if p.is_dir() and not p.name.startswith("__")
|
|
}
|
|
|
|
|
|
@when("I analyze package imports")
|
|
def step_analyze_imports(context):
|
|
"""Analyze imports in all Python files."""
|
|
context.imports = {}
|
|
for package_name, package_path in context.packages.items():
|
|
package_imports = set()
|
|
for py_file in package_path.rglob("*.py"):
|
|
if py_file.name == "__init__.py" and py_file.read_text().strip() == "":
|
|
continue
|
|
try:
|
|
tree = ast.parse(py_file.read_text())
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.startswith("cleveragents."):
|
|
package_imports.add(alias.name.split(".")[1])
|
|
elif (
|
|
isinstance(node, ast.ImportFrom)
|
|
and node.module
|
|
and node.module.startswith("cleveragents.")
|
|
):
|
|
package_imports.add(node.module.split(".")[1])
|
|
except SyntaxError:
|
|
pass # Skip files with syntax errors
|
|
context.imports[package_name] = package_imports
|
|
|
|
|
|
@then("{package} should not import from {target}")
|
|
def step_verify_no_import(context, package, target):
|
|
"""Verify a package doesn't import from another."""
|
|
if package not in context.imports:
|
|
return # Package doesn't exist yet
|
|
imports = context.imports.get(package, set())
|
|
assert target not in imports, f"{package} imports from {target}"
|
|
|
|
|
|
@then("the core package should be self-contained")
|
|
def step_verify_core_self_contained(context):
|
|
"""Verify core package doesn't import from any other internal package."""
|
|
package = "core"
|
|
if package not in context.imports:
|
|
return # Package doesn't exist yet
|
|
imports = context.imports.get(package, set())
|
|
# Filter out self-imports and allowed imports
|
|
# Core can import from config for dependency injection container setup
|
|
# and from shared for cross-cutting utilities (redaction, etc.)
|
|
allowed_imports = {"core", "config", "shared"}
|
|
other_imports = imports - allowed_imports
|
|
assert not other_imports, f"{package} imports from: {other_imports}"
|
|
|
|
|
|
@given("the settings module exists")
|
|
def step_settings_module_exists(context):
|
|
"""Check if settings module exists."""
|
|
context.settings_file = Path("src/cleveragents/config/settings.py")
|
|
assert context.settings_file.exists(), "Settings module does not exist"
|
|
|
|
|
|
@when("I check environment variable definitions")
|
|
def step_check_env_vars(context):
|
|
"""Extract environment variable definitions from settings."""
|
|
content = context.settings_file.read_text()
|
|
# Find all environment variable references
|
|
context.env_vars = re.findall(r'["\']([A-Z_]+)["\']', content)
|
|
|
|
|
|
@then("all application variables should use {prefix} prefix")
|
|
def step_verify_env_prefix(context, prefix):
|
|
"""Verify application environment variables use correct prefix."""
|
|
# Provider prefixes that should be excluded
|
|
provider_prefixes = [
|
|
"OPENAI_",
|
|
"ANTHROPIC_",
|
|
"CLAUDE_",
|
|
"GEMINI_",
|
|
"AZURE_",
|
|
"GOOGLE_",
|
|
"OPENROUTER_",
|
|
"HF_",
|
|
"HUGGINGFACEHUB_",
|
|
"HUGGING_FACE_HUB_",
|
|
"COHERE_",
|
|
"PERPLEXITY_",
|
|
"GROQ_",
|
|
"TOGETHER_",
|
|
]
|
|
|
|
# Common constants and system variables to exclude
|
|
excluded_constants = [
|
|
"LANGCHAIN_PROJECT_TYPE", # LangChain internal constant
|
|
"LANGCHAIN_CHAT_MODEL", # LangChain testing constant
|
|
"LANGCHAIN_HANDLER", # LangChain handler prefix
|
|
"INFO", # Logging level constants pulled from settings
|
|
"DEBUG",
|
|
"WARNING",
|
|
"ERROR",
|
|
"CRITICAL",
|
|
]
|
|
|
|
langsmith_allowlist = {
|
|
# LangChain/LangSmith compatibility variables that Settings syncs automatically
|
|
"LANGCHAIN_TRACING_V2",
|
|
"LANGCHAIN_API_KEY",
|
|
"LANGCHAIN_PROJECT",
|
|
"LANGCHAIN_ENDPOINT",
|
|
"LANGSMITH_TRACING_V2",
|
|
"LANGSMITH_API_KEY",
|
|
"LANGSMITH_PROJECT",
|
|
"LANGSMITH_ENDPOINT",
|
|
"LANGSMITH_USER_ID",
|
|
}
|
|
|
|
app_vars = [
|
|
var
|
|
for var in context.env_vars
|
|
if not any(var.startswith(p) for p in provider_prefixes)
|
|
and var not in excluded_constants
|
|
]
|
|
|
|
invalid_vars = [
|
|
var
|
|
for var in app_vars
|
|
if not var.startswith(prefix) and var not in langsmith_allowlist
|
|
]
|
|
assert not invalid_vars, f"Variables not using {prefix} prefix: {invalid_vars}"
|
|
|
|
|
|
@then("provider variables should keep their original prefix")
|
|
def step_verify_provider_vars(context):
|
|
"""Verify provider variables keep their original prefix."""
|
|
provider_prefixes = [
|
|
"OPENAI_",
|
|
"ANTHROPIC_",
|
|
"CLAUDE_",
|
|
"GEMINI_",
|
|
"AZURE_",
|
|
"GOOGLE_",
|
|
"OPENROUTER_",
|
|
"HF_",
|
|
"HUGGINGFACEHUB_",
|
|
"HUGGING_FACE_HUB_",
|
|
"COHERE_",
|
|
"PERPLEXITY_",
|
|
"GROQ_",
|
|
"TOGETHER_",
|
|
]
|
|
provider_vars = [
|
|
var
|
|
for var in context.env_vars
|
|
if any(var.startswith(p) for p in provider_prefixes)
|
|
]
|
|
# Just verify they exist and aren't renamed
|
|
for var in provider_vars:
|
|
assert not var.startswith("CLEVERAGENTS_"), f"Provider var renamed: {var}"
|
|
|
|
|
|
@given("the source code exists")
|
|
def step_source_code_exists(context):
|
|
"""Set up source code for analysis."""
|
|
context.src_dir = Path("src/cleveragents")
|
|
assert context.src_dir.exists()
|
|
|
|
|
|
@when("I check for type annotations")
|
|
def step_check_type_annotations(context):
|
|
"""Analyze type annotations in the codebase."""
|
|
context.functions_with_hints = 0
|
|
context.functions_without_hints = 0
|
|
context.pydantic_models = 0
|
|
context.regular_dataclasses = 0
|
|
|
|
for py_file in context.src_dir.rglob("*.py"):
|
|
if "discovery" in str(py_file):
|
|
continue # Skip discovery module as per pyright config
|
|
|
|
try:
|
|
tree = ast.parse(py_file.read_text())
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.FunctionDef):
|
|
# Check if function has return type annotation
|
|
if node.returns is not None:
|
|
context.functions_with_hints += 1
|
|
else:
|
|
# Skip __init__, __str__, __repr__ methods
|
|
if not node.name.startswith("_"):
|
|
context.functions_without_hints += 1
|
|
|
|
elif isinstance(node, ast.ClassDef):
|
|
# Check if it's a Pydantic model or dataclass
|
|
for base in node.bases:
|
|
base_name = (
|
|
ast.unparse(base) if hasattr(ast, "unparse") else str(base)
|
|
)
|
|
if "BaseModel" in base_name or "BaseSettings" in base_name:
|
|
context.pydantic_models += 1
|
|
elif "dataclass" in [
|
|
d.id for d in node.decorator_list if isinstance(d, ast.Name)
|
|
]:
|
|
context.regular_dataclasses += 1
|
|
except Exception:
|
|
pass # Skip files with syntax errors
|
|
|
|
|
|
@then("all public functions should have type hints")
|
|
def step_verify_public_functions_type_hints(context):
|
|
"""Verify public functions have type hints."""
|
|
missing_annotations = []
|
|
for py_file in context.src_dir.rglob("*.py"):
|
|
try:
|
|
tree = ast.parse(py_file.read_text())
|
|
except SyntaxError:
|
|
continue
|
|
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.FunctionDef) and not node.name.startswith("_"):
|
|
args = node.args
|
|
has_annotations = True
|
|
for arg in args.args + args.kwonlyargs:
|
|
if arg.arg not in ("self", "cls") and arg.annotation is None:
|
|
has_annotations = False
|
|
break
|
|
if args.vararg and args.vararg.annotation is None:
|
|
has_annotations = False
|
|
if args.kwarg and args.kwarg.annotation is None:
|
|
has_annotations = False
|
|
if node.returns is None:
|
|
has_annotations = False
|
|
if not has_annotations:
|
|
missing_annotations.append(f"{py_file}:{node.name}")
|
|
|
|
assert not missing_annotations, (
|
|
"Public functions missing type hints:\n" + "\n".join(missing_annotations)
|
|
)
|
|
|
|
|
|
@then("all dataclasses should use Pydantic models")
|
|
def step_verify_dataclasses_pydantic(context):
|
|
"""Verify dataclasses are Pydantic models."""
|
|
missing_pydantic = []
|
|
for py_file in context.src_dir.rglob("*.py"):
|
|
try:
|
|
tree = ast.parse(py_file.read_text())
|
|
except SyntaxError:
|
|
continue
|
|
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ClassDef):
|
|
bases = [base.id for base in node.bases if isinstance(base, ast.Name)]
|
|
if "BaseModel" in bases:
|
|
continue
|
|
decorators = [
|
|
decorator.id
|
|
for decorator in node.decorator_list
|
|
if isinstance(decorator, ast.Name)
|
|
]
|
|
if "dataclass" in decorators:
|
|
missing_pydantic.append(f"{py_file}:{node.name}")
|
|
|
|
assert not missing_pydantic, (
|
|
"Dataclasses missing Pydantic BaseModel inheritance:\n"
|
|
+ "\n".join(missing_pydantic)
|
|
)
|
|
|
|
|
|
@when("I read the CONTRIBUTING guidelines")
|
|
def step_read_contributing_guidelines(context):
|
|
"""Load the CONTRIBUTING.md document."""
|
|
contributing_path = Path("CONTRIBUTING.md")
|
|
assert contributing_path.exists(), "CONTRIBUTING.md not found"
|
|
context.contributing_text = contributing_path.read_text(encoding="utf-8")
|
|
|
|
|
|
@then('the contributing document should include "{text}"')
|
|
def step_contributing_includes_text(context, text):
|
|
"""Ensure CONTRIBUTING.md contains the expected text."""
|
|
content = getattr(context, "contributing_text", "")
|
|
assert text in content, f"Expected '{text}' in CONTRIBUTING.md"
|
|
|
|
|
|
@when("I import every module under the source directory")
|
|
def step_import_every_module(context):
|
|
"""Import every module beneath src/cleveragents so coverage sees them."""
|
|
|
|
if not hasattr(context, "src_dir"):
|
|
context.src_dir = Path("src/cleveragents")
|
|
assert context.src_dir.exists(), "Source directory missing"
|
|
|
|
importlib.import_module("cleveragents")
|
|
|
|
context.module_import_errors = []
|
|
package_root = str(context.src_dir)
|
|
|
|
for module_info in pkgutil.walk_packages([package_root], prefix="cleveragents."):
|
|
module_name = module_info.name
|
|
if module_name.endswith(".__main__"):
|
|
continue
|
|
try:
|
|
importlib.import_module(module_name)
|
|
except Exception as exc: # pragma: no cover - only runs on failure
|
|
context.module_import_errors.append((module_name, repr(exc)))
|
|
|
|
|
|
@then("every module should import without errors")
|
|
def step_assert_module_imports(context):
|
|
"""Fail if any module imports raised an exception."""
|
|
|
|
errors = getattr(context, "module_import_errors", [])
|
|
assert not errors, "Some modules failed to import:\n" + "\n".join(
|
|
f"- {name}: {error}" for name, error in errors
|
|
)
|