freemo
7b1020e735
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 18s
CI / typecheck (pull_request) Successful in 36s
CI / security (pull_request) Successful in 36s
CI / unit_tests (pull_request) Successful in 3m46s
CI / docker (pull_request) Successful in 39s
CI / integration_tests (pull_request) Successful in 4m30s
CI / coverage (pull_request) Successful in 4m35s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 16s
CI / quality (push) Successful in 16s
CI / security (push) Successful in 33s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m7s
CI / docker (push) Successful in 39s
CI / integration_tests (push) Successful in 3m4s
CI / coverage (push) Successful in 4m35s
CI / benchmark-publish (push) Has been cancelled
CI / benchmark-regression (pull_request) Successful in 29m37s
Implement spec-mandated prompt injection protections: 1. Input sanitization (PromptSanitizer.sanitize_user_input): escapes HTML entities, strips C0/C1 control characters, rejects 8 known injection patterns including instruction override, role assumption, ChatML tags, and boundary marker spoofing. 2. Prompt boundary markers (PromptSanitizer.wrap_user_content): wraps user content with [USER_CONTENT_START]/[USER_CONTENT_END] markers; augment_system_prompt() prepends boundary recognition instructions. 3. Output validation: existing schema_validator.py validates tool I/O against JSON Schema in ToolRuntime.execute() (verified, tested). 4. Tool capability restrictions: existing _enforce_capabilities() in lifecycle.py enforces read_only/writes/checkpointable/side_effects declarations (verified, tested). 5. Unsafe tool gating: existing _enforce_capabilities() blocks unsafe tools unless allow_unsafe_tools=true in automation profile (verified, tested). Integrates sanitizer into session prompt construction, invariant text processing, and action argument handling paths. ISSUES CLOSED: #572
124 lines
3.8 KiB
Python
124 lines
3.8 KiB
Python
"""Prompt Injection Mitigation sanitization throughput benchmarks.
|
|
|
|
Measures the performance of:
|
|
- PromptSanitizer.sanitize_user_input (clean and dirty text)
|
|
- PromptSanitizer.wrap_user_content
|
|
- PromptSanitizer.augment_system_prompt
|
|
- PromptSanitizer.sanitize_and_wrap
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.application.services.prompt_sanitizer import PromptSanitizer
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.application.services.prompt_sanitizer import PromptSanitizer
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Clean input suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeSanitizeCleanInput:
|
|
"""Benchmark sanitization of clean (benign) input."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare sanitizer and clean text."""
|
|
self.sanitizer = PromptSanitizer()
|
|
self.clean_text = (
|
|
"Please refactor the authentication module to use OAuth2 " * 100
|
|
)
|
|
|
|
def time_sanitize_clean_input(self) -> None:
|
|
"""Benchmark sanitize_user_input on clean text."""
|
|
self.sanitizer.sanitize_user_input(self.clean_text)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dirty input suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeSanitizeDirtyInput:
|
|
"""Benchmark sanitization of dirty (control chars + HTML entities) input."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare sanitizer and dirty text."""
|
|
self.sanitizer = PromptSanitizer()
|
|
self.dirty_text = (
|
|
"Hello <world> & 'friends' with \x00\x01\x02 control chars " * 100
|
|
)
|
|
|
|
def time_sanitize_dirty_input(self) -> None:
|
|
"""Benchmark sanitize_user_input on dirty text."""
|
|
self.sanitizer.sanitize_user_input(self.dirty_text)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wrap content suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeWrapContent:
|
|
"""Benchmark wrap_user_content throughput."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare sanitizer and content."""
|
|
self.sanitizer = PromptSanitizer()
|
|
self.text = "Some user content " * 500
|
|
|
|
def time_wrap_user_content(self) -> None:
|
|
"""Benchmark wrap_user_content."""
|
|
self.sanitizer.wrap_user_content(self.text)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Augment system prompt suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeAugmentSystemPrompt:
|
|
"""Benchmark augment_system_prompt throughput."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare sanitizer and prompt."""
|
|
self.sanitizer = PromptSanitizer()
|
|
self.prompt = "You are a helpful coding assistant. " * 50
|
|
|
|
def time_augment_system_prompt(self) -> None:
|
|
"""Benchmark augment_system_prompt."""
|
|
self.sanitizer.augment_system_prompt(self.prompt)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sanitize-and-wrap suite
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeSanitizeAndWrap:
|
|
"""Benchmark sanitize_and_wrap (combined convenience method)."""
|
|
|
|
timeout = 30
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare sanitizer and text."""
|
|
self.sanitizer = PromptSanitizer()
|
|
self.text = "Refactor authentication to use <OAuth2> & JWT tokens " * 50
|
|
|
|
def time_sanitize_and_wrap(self) -> None:
|
|
"""Benchmark sanitize_and_wrap."""
|
|
self.sanitizer.sanitize_and_wrap(self.text)
|