forked from HAL9000/cleveragents-core
239 lines
9.6 KiB
Python
239 lines
9.6 KiB
Python
"""
|
|
Unit tests for core/sandbox.py
|
|
|
|
Tests the SAFE_BUILTINS dictionary used for sandboxed execution.
|
|
"""
|
|
|
|
import os
|
|
import pytest
|
|
import tempfile
|
|
|
|
from cleveragents.core.sandbox import SAFE_BUILTINS
|
|
|
|
|
|
class TestSafeBuiltins:
|
|
"""Test suite for SAFE_BUILTINS."""
|
|
|
|
def test_safe_builtins_is_dict(self):
|
|
"""Test that SAFE_BUILTINS is a dictionary."""
|
|
assert isinstance(SAFE_BUILTINS, dict)
|
|
|
|
def test_safe_builtins_not_empty(self):
|
|
"""Test that SAFE_BUILTINS is not empty."""
|
|
assert len(SAFE_BUILTINS) > 0
|
|
|
|
def test_basic_types_included(self):
|
|
"""Test that basic Python types are included."""
|
|
expected_types = ["bool", "dict", "float", "int", "list", "set", "str", "tuple"]
|
|
|
|
for type_name in expected_types:
|
|
assert type_name in SAFE_BUILTINS, f"{type_name} should be in SAFE_BUILTINS"
|
|
|
|
def test_basic_types_are_correct_builtins(self):
|
|
"""Test that basic types map to correct built-in types."""
|
|
assert SAFE_BUILTINS["bool"] is bool
|
|
assert SAFE_BUILTINS["dict"] is dict
|
|
assert SAFE_BUILTINS["float"] is float
|
|
assert SAFE_BUILTINS["int"] is int
|
|
assert SAFE_BUILTINS["list"] is list
|
|
assert SAFE_BUILTINS["set"] is set
|
|
assert SAFE_BUILTINS["str"] is str
|
|
assert SAFE_BUILTINS["tuple"] is tuple
|
|
|
|
def test_safe_utility_functions_included(self):
|
|
"""Test that safe utility functions are included."""
|
|
expected_functions = ["abs", "all", "any", "len", "max", "min", "round", "sum"]
|
|
|
|
for func_name in expected_functions:
|
|
assert func_name in SAFE_BUILTINS, f"{func_name} should be in SAFE_BUILTINS"
|
|
|
|
def test_safe_utility_functions_are_correct_builtins(self):
|
|
"""Test that utility functions map to correct built-in functions."""
|
|
assert SAFE_BUILTINS["abs"] is abs
|
|
assert SAFE_BUILTINS["all"] is all
|
|
assert SAFE_BUILTINS["any"] is any
|
|
assert SAFE_BUILTINS["len"] is len
|
|
assert SAFE_BUILTINS["max"] is max
|
|
assert SAFE_BUILTINS["min"] is min
|
|
assert SAFE_BUILTINS["round"] is round
|
|
assert SAFE_BUILTINS["sum"] is sum
|
|
|
|
def test_dangerous_functions_not_included(self):
|
|
"""Test that dangerous functions are NOT included."""
|
|
dangerous = ["open", "eval", "exec", "compile", "__import__", "input", "print"]
|
|
|
|
for func_name in dangerous:
|
|
assert func_name not in SAFE_BUILTINS, f"{func_name} should NOT be in SAFE_BUILTINS"
|
|
|
|
def test_introspection_functions_not_included(self):
|
|
"""Test that introspection functions are NOT included."""
|
|
introspection = ["dir", "vars", "globals", "locals", "getattr", "setattr", "hasattr"]
|
|
|
|
for func_name in introspection:
|
|
assert func_name not in SAFE_BUILTINS, f"{func_name} should NOT be in SAFE_BUILTINS"
|
|
|
|
def test_safe_builtins_can_be_used_in_eval(self):
|
|
"""Test that SAFE_BUILTINS can be used safely in eval."""
|
|
# Basic arithmetic
|
|
result = eval("abs(-5)", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 5
|
|
|
|
# Type conversions
|
|
result = eval("int('42')", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 42
|
|
|
|
# Collection operations
|
|
result = eval("len([1, 2, 3])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 3
|
|
|
|
# Aggregations
|
|
result = eval("sum([1, 2, 3, 4])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 10
|
|
|
|
def test_safe_builtins_prevents_dangerous_operations(self):
|
|
"""Test that dangerous operations fail with SAFE_BUILTINS."""
|
|
# Should raise NameError because 'open' is not in SAFE_BUILTINS
|
|
with pytest.raises(NameError):
|
|
eval("open('/etc/passwd')", {"__builtins__": SAFE_BUILTINS})
|
|
|
|
# Should raise NameError because 'eval' is not in SAFE_BUILTINS
|
|
with pytest.raises(NameError):
|
|
eval("eval('1+1')", {"__builtins__": SAFE_BUILTINS})
|
|
|
|
# Should raise NameError because '__import__' is not in SAFE_BUILTINS
|
|
with pytest.raises(NameError):
|
|
eval("__import__('os')", {"__builtins__": SAFE_BUILTINS})
|
|
|
|
def test_safe_builtins_prevents_execution_not_just_detection(self):
|
|
"""Test that dangerous operations are PREVENTED, not executed then caught.
|
|
|
|
This is a critical security test per Brent's feedback. SAFE_BUILTINS must
|
|
prevent dangerous code from executing at all, not just detect it after
|
|
execution.
|
|
|
|
Context: If eval() executed code before raising NameError, an attacker could:
|
|
- Create/delete files (disk space attacks like dd if=/dev/zero of=/tmp/file)
|
|
- Access sensitive data
|
|
- Consume resources
|
|
|
|
This test verifies that NameError is raised BEFORE any dangerous operation
|
|
executes, proving SAFE_BUILTINS provides prevention, not just detection.
|
|
"""
|
|
# Test 1: Verify file creation is prevented (not just detected)
|
|
temp_path = tempfile.mktemp(suffix='.sandbox_security_test')
|
|
|
|
try:
|
|
# This should raise NameError without creating the file
|
|
with pytest.raises(NameError, match="name 'open' is not defined"):
|
|
eval(f"open('{temp_path}', 'w').write('test')",
|
|
{"__builtins__": SAFE_BUILTINS})
|
|
|
|
# CRITICAL SECURITY CHECK: File should never have been created
|
|
assert not os.path.exists(temp_path), \
|
|
"SECURITY VULNERABILITY: File was created before NameError was raised! " \
|
|
"SAFE_BUILTINS is not preventing execution, only detecting it."
|
|
finally:
|
|
# Cleanup if file somehow exists (indicates test failure)
|
|
if os.path.exists(temp_path):
|
|
os.unlink(temp_path)
|
|
pytest.fail("SECURITY ISSUE: Temporary file was created, indicating "
|
|
"dangerous code executed before being caught")
|
|
|
|
# Test 2: Verify __import__ is prevented before module import
|
|
# If this were to execute, it would import os module and execute system command
|
|
with pytest.raises(NameError, match="name '__import__' is not defined"):
|
|
eval("__import__('os').system('echo SECURITY_BREACH')",
|
|
{"__builtins__": SAFE_BUILTINS})
|
|
|
|
# Test 3: Verify nested eval is prevented at the first eval call
|
|
with pytest.raises(NameError, match="name 'eval' is not defined"):
|
|
eval("eval('__import__(\"os\")')",
|
|
{"__builtins__": SAFE_BUILTINS})
|
|
|
|
# Test 4: Verify exec is prevented
|
|
with pytest.raises(NameError, match="name 'exec' is not defined"):
|
|
eval("exec('import os')",
|
|
{"__builtins__": SAFE_BUILTINS})
|
|
|
|
def test_safe_builtins_with_complex_expressions(self):
|
|
"""Test SAFE_BUILTINS with more complex safe expressions."""
|
|
# Nested list comprehension with safe operations
|
|
result = eval(
|
|
"sum([abs(x) for x in [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4]])",
|
|
{"__builtins__": SAFE_BUILTINS}
|
|
)
|
|
assert result == 25 # 5+4+3+2+1+0+1+2+3+4
|
|
|
|
# Dictionary and set operations
|
|
result = eval(
|
|
"len(set([1, 2, 2, 3, 3, 3]))",
|
|
{"__builtins__": SAFE_BUILTINS}
|
|
)
|
|
assert result == 3
|
|
|
|
# String operations
|
|
result = eval(
|
|
"str(42) + '_test'",
|
|
{"__builtins__": SAFE_BUILTINS}
|
|
)
|
|
assert result == "42_test"
|
|
|
|
def test_safe_builtins_all_values_are_callable_or_types(self):
|
|
"""Test that all values in SAFE_BUILTINS are callable or types."""
|
|
for name, value in SAFE_BUILTINS.items():
|
|
assert callable(value) or isinstance(value, type), \
|
|
f"{name} should be callable or a type"
|
|
|
|
def test_safe_builtins_count(self):
|
|
"""Test that SAFE_BUILTINS has the expected number of entries."""
|
|
# 8 types + 8 utility functions = 16 total
|
|
assert len(SAFE_BUILTINS) == 16
|
|
|
|
def test_safe_builtins_all_function(self):
|
|
"""Test the 'all' function behavior."""
|
|
result = eval("all([True, True, True])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result is True
|
|
|
|
result = eval("all([True, False, True])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result is False
|
|
|
|
def test_safe_builtins_any_function(self):
|
|
"""Test the 'any' function behavior."""
|
|
result = eval("any([False, False, False])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result is False
|
|
|
|
result = eval("any([False, True, False])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result is True
|
|
|
|
def test_safe_builtins_max_min_functions(self):
|
|
"""Test the 'max' and 'min' functions."""
|
|
result = eval("max([1, 5, 3, 9, 2])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 9
|
|
|
|
result = eval("min([1, 5, 3, 9, 2])", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 1
|
|
|
|
def test_safe_builtins_round_function(self):
|
|
"""Test the 'round' function behavior."""
|
|
result = eval("round(3.7)", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 4
|
|
|
|
result = eval("round(3.14159, 2)", {"__builtins__": SAFE_BUILTINS})
|
|
assert result == 3.14
|
|
|
|
def test_safe_builtins_immutable(self):
|
|
"""Test that SAFE_BUILTINS dictionary itself should not be modified."""
|
|
original_len = len(SAFE_BUILTINS)
|
|
original_keys = set(SAFE_BUILTINS.keys())
|
|
|
|
# Attempt to use it should not modify the original
|
|
eval("abs(-10)", {"__builtins__": SAFE_BUILTINS})
|
|
|
|
assert len(SAFE_BUILTINS) == original_len
|
|
assert set(SAFE_BUILTINS.keys()) == original_keys
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|
|
|