CoreRasurae
66b9a4279f
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 17s
CI / security (pull_request) Successful in 32s
CI / typecheck (pull_request) Successful in 35s
CI / unit_tests (pull_request) Successful in 1m55s
CI / docker (pull_request) Successful in 39s
CI / integration_tests (pull_request) Successful in 2m50s
CI / coverage (pull_request) Successful in 4m13s
CI / benchmark-regression (pull_request) Successful in 23m10s
CI / lint (push) Successful in 12s
CI / quality (push) Successful in 15s
CI / build (push) Successful in 15s
CI / security (push) Successful in 29s
CI / typecheck (push) Successful in 32s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 3m10s
CI / docker (push) Successful in 38s
CI / integration_tests (push) Successful in 4m1s
CI / coverage (push) Successful in 3m58s
CI / benchmark-publish (push) Successful in 13m21s
Add SafetyProfile domain model with configurable safety constraints (checkpoint, human approval, cost limits, retry limits, skill categories, sandbox requirement). Integrate into Action model with from_config/as_cli_dict support, and persist via LifecycleActionModel safety_profile_json column. Include resolve_safety_profile() stub that raises NotImplementedError for local mode, signalling that real enforcement is deferred. Add comprehensive test coverage: - 20 BDD scenarios in features/safety_profile.feature - 6 Robot Framework smoke tests - 5 ASV benchmark suites (11 timing functions) Add docs/reference/safety_profile.md reference documentation. ISSUES CLOSED: #332
134 lines
4.0 KiB
Python
134 lines
4.0 KiB
Python
"""ASV benchmarks for SafetyProfile validation overhead.
|
|
|
|
Measures the performance of:
|
|
- SafetyProfile model construction (Pydantic validation)
|
|
- SafetyProfile.model_dump() serialization
|
|
- SafetyProfile.model_dump_json() JSON serialization
|
|
- SafetyProfile.from_config() factory
|
|
- DEFAULT_SAFETY_PROFILE access
|
|
- SafetyProfileRef construction
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.safety_profile import (
|
|
DEFAULT_SAFETY_PROFILE,
|
|
SafetyProfile,
|
|
SafetyProfileProvenance,
|
|
SafetyProfileRef,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.domain.models.core.safety_profile import (
|
|
DEFAULT_SAFETY_PROFILE,
|
|
SafetyProfile,
|
|
SafetyProfileProvenance,
|
|
SafetyProfileRef,
|
|
)
|
|
|
|
|
|
def _make_full_profile() -> SafetyProfile:
|
|
"""Create a fully-populated safety profile for benchmarking."""
|
|
return SafetyProfile(
|
|
allowed_skill_categories=["code", "test", "deploy", "review"],
|
|
require_sandbox=True,
|
|
require_checkpoints=True,
|
|
require_human_approval=True,
|
|
allow_unsafe_tools=False,
|
|
max_cost_per_plan=100.0,
|
|
max_retries_per_step=5,
|
|
max_total_cost=500.0,
|
|
)
|
|
|
|
|
|
class SafetyProfileValidationSuite:
|
|
"""Benchmark SafetyProfile construction."""
|
|
|
|
def time_default_construction(self) -> None:
|
|
"""Benchmark default SafetyProfile creation."""
|
|
SafetyProfile()
|
|
|
|
def time_full_construction(self) -> None:
|
|
"""Benchmark fully-populated SafetyProfile creation."""
|
|
_make_full_profile()
|
|
|
|
def time_minimal_construction(self) -> None:
|
|
"""Benchmark minimal SafetyProfile creation."""
|
|
SafetyProfile(require_sandbox=False)
|
|
|
|
def time_with_categories(self) -> None:
|
|
"""Benchmark SafetyProfile with skill categories."""
|
|
SafetyProfile(
|
|
allowed_skill_categories=["code", "test", "deploy"],
|
|
)
|
|
|
|
|
|
class SafetyProfileSerializationSuite:
|
|
"""Benchmark SafetyProfile serialization."""
|
|
|
|
def setup(self) -> None:
|
|
"""Create objects for serialization benchmarks."""
|
|
self.profile = _make_full_profile()
|
|
|
|
def time_model_dump(self) -> None:
|
|
"""Benchmark model_dump() serialization."""
|
|
self.profile.model_dump()
|
|
|
|
def time_model_dump_json(self) -> None:
|
|
"""Benchmark model_dump_json() JSON serialization."""
|
|
self.profile.model_dump_json()
|
|
|
|
|
|
class SafetyProfileFromConfigSuite:
|
|
"""Benchmark SafetyProfile.from_config() factory."""
|
|
|
|
def setup(self) -> None:
|
|
"""Prepare config dicts for benchmarks."""
|
|
self.config = {
|
|
"require_sandbox": False,
|
|
"require_checkpoints": True,
|
|
"require_human_approval": True,
|
|
"max_cost_per_plan": 50.0,
|
|
"max_retries_per_step": 3,
|
|
"allowed_skill_categories": ["code", "test"],
|
|
}
|
|
|
|
def time_from_config(self) -> None:
|
|
"""Benchmark from_config()."""
|
|
SafetyProfile.from_config(self.config)
|
|
|
|
|
|
class SafetyProfileDefaultSuite:
|
|
"""Benchmark DEFAULT_SAFETY_PROFILE access."""
|
|
|
|
def time_access_default(self) -> None:
|
|
"""Benchmark accessing the default constant."""
|
|
_ = DEFAULT_SAFETY_PROFILE
|
|
|
|
def time_default_dump(self) -> None:
|
|
"""Benchmark dumping the default profile."""
|
|
DEFAULT_SAFETY_PROFILE.model_dump()
|
|
|
|
|
|
class SafetyProfileRefSuite:
|
|
"""Benchmark SafetyProfileRef construction."""
|
|
|
|
def time_ref_construction(self) -> None:
|
|
"""Benchmark SafetyProfileRef creation."""
|
|
SafetyProfileRef(
|
|
profile_name="strict",
|
|
provenance=SafetyProfileProvenance.PLAN,
|
|
)
|
|
|
|
def time_ref_all_provenances(self) -> None:
|
|
"""Benchmark creating refs with all provenance levels."""
|
|
for prov in SafetyProfileProvenance:
|
|
SafetyProfileRef(
|
|
profile_name="test",
|
|
provenance=prov,
|
|
)
|