Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 927c1f0e4b | |||
| cb28ecf63d |
@@ -43,6 +43,13 @@
|
||||
method. CLI `plan status` shows per-project changeset summaries for multi-project plans.
|
||||
Includes Behave BDD scenarios, Robot Framework smoke tests, ASV benchmarks, and reference
|
||||
documentation. (#199)
|
||||
- Added safety profile enforcement test coverage: 12 Behave BDD scenarios for boolean flag
|
||||
toggles, deny-none semantics, cost-without-total, type validation, negative-cost rejection,
|
||||
upper-bound retries, and restrictive full-constraint profiles; 8 cost/retry
|
||||
Action-integration scenarios including equal-value boundary; 2 Robot Framework
|
||||
smoke tests (validation rules, action attachment); and ASV benchmarks for test-scenario
|
||||
runtime baselines. Updated `docs/development/testing.md` with safety profile test fixture
|
||||
documentation. (#333)
|
||||
- Added large-project hierarchical decomposition with 4+ levels and bounded context per
|
||||
subplan. Includes clustering heuristics (directory, language, size), bounded dependency
|
||||
closure with memoization for 10K+ files, DAG execution ordering with cycle detection,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""ASV benchmarks for SafetyProfile domain model scenario runtime.
|
||||
|
||||
Measures:
|
||||
- SafetyProfile construction time (defaults and explicit fields)
|
||||
- Cross-field cost validation overhead
|
||||
- Skill category deduplication overhead
|
||||
- SafetyProfile serialization / deserialization round-trip
|
||||
- Action with safety profile attachment construction
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cleveragents.domain.models.core.safety_profile import SafetyProfile
|
||||
except ModuleNotFoundError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
from cleveragents.domain.models.core.safety_profile import SafetyProfile
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_safety_profile() -> SafetyProfile:
|
||||
"""Create a fully-populated SafetyProfile."""
|
||||
return SafetyProfile(
|
||||
require_sandbox=True,
|
||||
require_checkpoints=True,
|
||||
allow_unsafe_tools=False,
|
||||
require_human_approval=True,
|
||||
allowed_skill_categories=["code", "data", "infra"],
|
||||
max_cost_per_plan=50.0,
|
||||
max_total_cost=200.0,
|
||||
max_retries_per_step=5,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Construction benchmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SafetyProfileConstructionSuite:
|
||||
"""Benchmark SafetyProfile construction variants."""
|
||||
|
||||
def time_default_construction(self) -> None:
|
||||
"""Measure default SafetyProfile construction time."""
|
||||
SafetyProfile()
|
||||
|
||||
def time_full_construction(self) -> None:
|
||||
"""Measure fully-populated SafetyProfile construction time."""
|
||||
_make_safety_profile()
|
||||
|
||||
def time_minimal_construction_overrides(self) -> None:
|
||||
"""Measure SafetyProfile with minimal overrides."""
|
||||
SafetyProfile(require_sandbox=False, allow_unsafe_tools=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation benchmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SafetyProfileValidationSuite:
|
||||
"""Benchmark SafetyProfile validation operations."""
|
||||
|
||||
def time_cost_bounds_validation_valid(self) -> None:
|
||||
"""Measure valid cost bounds validation."""
|
||||
SafetyProfile(max_cost_per_plan=50.0, max_total_cost=100.0)
|
||||
|
||||
def time_cost_bounds_validation_equal(self) -> None:
|
||||
"""Measure cost bounds validation with equal values."""
|
||||
SafetyProfile(max_cost_per_plan=100.0, max_total_cost=100.0)
|
||||
|
||||
def time_retry_bounds_validation(self) -> None:
|
||||
"""Measure retry bounds validation."""
|
||||
SafetyProfile(max_retries_per_step=50)
|
||||
|
||||
def time_skill_category_deduplication(self) -> None:
|
||||
"""Measure skill category trimming and deduplication."""
|
||||
SafetyProfile(
|
||||
allowed_skill_categories=[
|
||||
"code",
|
||||
" code ",
|
||||
"data",
|
||||
"code",
|
||||
"infra",
|
||||
" data ",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialization benchmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SafetyProfileSerializationSuite:
|
||||
"""Benchmark SafetyProfile serialization round-trips."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Create a profile for serialization benchmarks."""
|
||||
self.profile = _make_safety_profile()
|
||||
self.profile_dict = self.profile.model_dump()
|
||||
self.profile_json = self.profile.model_dump_json()
|
||||
|
||||
def time_model_dump(self) -> None:
|
||||
"""Measure SafetyProfile to dict serialization."""
|
||||
self.profile.model_dump()
|
||||
|
||||
def time_model_dump_json(self) -> None:
|
||||
"""Measure SafetyProfile to JSON serialization."""
|
||||
self.profile.model_dump_json()
|
||||
|
||||
def time_model_validate_from_dict(self) -> None:
|
||||
"""Measure SafetyProfile from dict deserialization."""
|
||||
SafetyProfile.model_validate(self.profile_dict)
|
||||
|
||||
def time_model_validate_json(self) -> None:
|
||||
"""Measure SafetyProfile from JSON deserialization."""
|
||||
SafetyProfile.model_validate_json(self.profile_json)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action attachment benchmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SafetyProfileActionAttachmentSuite:
|
||||
"""Benchmark Action construction with SafetyProfile attachment."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Pre-build a safety profile and import Action/NamespacedName."""
|
||||
from cleveragents.domain.models.core.action import Action
|
||||
from cleveragents.domain.models.core.plan import NamespacedName
|
||||
|
||||
self.Action = Action
|
||||
self.NamespacedName = NamespacedName
|
||||
self.safety_profile = _make_safety_profile()
|
||||
|
||||
def time_action_with_safety_profile(self) -> None:
|
||||
"""Measure Action construction with safety profile attached."""
|
||||
nn = self.NamespacedName(namespace="bench", name="safety-action")
|
||||
self.Action(
|
||||
namespaced_name=nn,
|
||||
description="Benchmark action with safety profile",
|
||||
definition_of_done="Benchmark complete",
|
||||
strategy_actor="bench/strategy",
|
||||
execution_actor="bench/execution",
|
||||
safety_profile=self.safety_profile,
|
||||
)
|
||||
|
||||
def time_action_without_safety_profile(self) -> None:
|
||||
"""Measure Action construction without safety profile."""
|
||||
nn = self.NamespacedName(namespace="bench", name="no-safety")
|
||||
self.Action(
|
||||
namespaced_name=nn,
|
||||
description="Benchmark action without safety profile",
|
||||
definition_of_done="Benchmark complete",
|
||||
strategy_actor="bench/strategy",
|
||||
execution_actor="bench/execution",
|
||||
)
|
||||
@@ -1507,4 +1507,51 @@ nox
|
||||
|
||||
---
|
||||
|
||||
## Safety Profile Test Fixtures
|
||||
|
||||
### Behave Feature Files
|
||||
|
||||
- **`features/safety_profile.feature`** — Core model validation: defaults,
|
||||
constraint validation, cross-field cost bounds, category sanitization,
|
||||
immutability, `from_config`/`from_yaml` factories, `SafetyProfileRef`,
|
||||
`resolve_safety_profile` stub, `model_dump` round-trip, Action integration,
|
||||
boolean flag toggles, and restrictive full-constraint profiles.
|
||||
- **`features/safety_profile_cost_retry.feature`** — Action-level cost/retry
|
||||
bounds: attaching `SafetyProfile` to `Action`, rejecting cross-field
|
||||
violations through action creation, valid/zero boundaries, and absent
|
||||
safety profile semantics.
|
||||
|
||||
### YAML Config Example
|
||||
|
||||
```yaml
|
||||
# Example safety profile YAML for test fixtures
|
||||
require_sandbox: true
|
||||
require_checkpoints: true
|
||||
require_human_approval: false
|
||||
allow_unsafe_tools: false
|
||||
max_cost_per_plan: 50.0
|
||||
max_total_cost: 200.0
|
||||
max_retries_per_step: 5
|
||||
allowed_skill_categories:
|
||||
- code
|
||||
- test
|
||||
- deploy
|
||||
```
|
||||
|
||||
### Robot Framework
|
||||
|
||||
- **`robot/safety_profile.robot`** — Smoke tests: default profile, config
|
||||
loading, YAML loading, serialization, `SafetyProfileRef`, constraint
|
||||
validation, Action attachment, and summary.
|
||||
|
||||
### ASV Benchmarks
|
||||
|
||||
- **`benchmarks/safety_profile_model_bench.py`** — Model construction,
|
||||
validation overhead, `from_yaml`, serialization, resolution stub.
|
||||
- **`benchmarks/safety_profile_tests_bench.py`** — Test-scenario runtime:
|
||||
construction variants, cost/retry validation, category deduplication,
|
||||
serialization round-trips, Action attachment overhead.
|
||||
|
||||
---
|
||||
|
||||
##
|
||||
|
||||
@@ -226,3 +226,80 @@ Feature: Safety Profile Domain Model
|
||||
Scenario: Safety profile from_yaml rejects non-dict content
|
||||
When I try to load a safety profile from yaml with non-dict content
|
||||
Then a safety ValueError should be raised
|
||||
|
||||
# ---- Boolean flag toggles (#333) ----
|
||||
|
||||
Scenario: Safety profile accepts sandbox disabled
|
||||
When I create a safety profile with require_sandbox false
|
||||
Then the safety profile should be created
|
||||
And the safety require_sandbox should be false
|
||||
|
||||
Scenario: Safety profile accepts checkpoints disabled
|
||||
When I create a safety profile with require_checkpoints false
|
||||
Then the safety profile should be created
|
||||
And the safety require_checkpoints should be false
|
||||
|
||||
Scenario: Safety profile accepts unsafe tools enabled
|
||||
When I create a safety profile with allow_unsafe_tools true
|
||||
Then the safety profile should be created
|
||||
And the safety allow_unsafe_tools should be true
|
||||
|
||||
Scenario: Safety profile accepts human approval required
|
||||
When I create a safety profile with require_human_approval true
|
||||
Then the safety profile should be created
|
||||
And the safety require_human_approval should be true
|
||||
|
||||
# ---- Deny-none semantics (#333) ----
|
||||
|
||||
Scenario: Empty skill categories list means all categories allowed
|
||||
When I create a default safety profile
|
||||
Then the safety profile should be created
|
||||
And the safety allowed_skill_categories should be empty
|
||||
|
||||
# ---- Cost per plan without total cost (#333) ----
|
||||
|
||||
Scenario: Safety profile allows cost per plan without total cost
|
||||
When I create a safety profile with max_cost_per_plan 50.0
|
||||
Then the safety profile should be created
|
||||
And the safety max_cost_per_plan should be 50.0
|
||||
And the safety max_total_cost should be none
|
||||
|
||||
Scenario: Safety profile allows total cost without cost per plan
|
||||
When I create a safety profile with max_total_cost 200.0
|
||||
Then the safety profile should be created
|
||||
And the safety max_total_cost should be 200.0
|
||||
And the safety max_cost_per_plan should be none
|
||||
|
||||
# ---- Type validation (#333) ----
|
||||
|
||||
Scenario: Safety profile rejects cost per plan as string
|
||||
When I try to create a safety profile with string cost
|
||||
Then a safety validation error should be raised
|
||||
|
||||
# ---- Negative cost rejection (#333/F-07) ----
|
||||
|
||||
Scenario: Safety profile rejects negative cost per plan via profile
|
||||
When I try to create a safety profile with max_cost_per_plan -5.0
|
||||
Then a safety validation error should be raised
|
||||
|
||||
# ---- Boundary value tests for max_retries (#333/F-09) ----
|
||||
|
||||
Scenario: Safety profile accepts max_retries_per_step at upper bound 100
|
||||
When I create a safety profile with max_retries_per_step 100
|
||||
Then the safety profile should be created
|
||||
And the safety max_retries_per_step should be 100
|
||||
|
||||
# ---- Restrictive full integration (#333) ----
|
||||
|
||||
Scenario: Restrictive safety profile validates all constraints
|
||||
When I create a restrictive safety profile
|
||||
Then the safety profile should be created
|
||||
And the safety require_sandbox should be true
|
||||
And the safety require_checkpoints should be true
|
||||
And the safety require_human_approval should be true
|
||||
And the safety allow_unsafe_tools should be false
|
||||
And the safety max_cost_per_plan should be 10.0
|
||||
And the safety max_total_cost should be 100.0
|
||||
And the safety max_retries_per_step should be 1
|
||||
And the safety allowed_skill_categories count should be 2
|
||||
And the safety allowed_skill_categories should contain "code" and "test"
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
Feature: Safety Profile cost and retry bounds on action creation
|
||||
Validates that cost and retry bounds on SafetyProfile are enforced
|
||||
during action creation when a safety_profile is attached to an Action.
|
||||
|
||||
# ============================================================
|
||||
# Action with safety profile cost bounds
|
||||
# ============================================================
|
||||
|
||||
Scenario: Action accepts safety profile with valid cost bounds
|
||||
Given a safety profile with max_cost_per_plan 50.0 and max_total_cost 200.0
|
||||
When I create an action with the safety profile attached
|
||||
Then the action with safety profile should be created successfully
|
||||
And the action safety profile max_cost_per_plan should be 50.0
|
||||
And the action safety profile max_total_cost should be 200.0
|
||||
|
||||
Scenario: Action rejects safety profile where cost per plan exceeds total
|
||||
When I try to create an action with safety profile cost_per_plan 300.0 and total 100.0
|
||||
Then an action safety profile validation error should be raised
|
||||
And the action safety profile error should mention "max_cost_per_plan"
|
||||
|
||||
# ============================================================
|
||||
# Action with safety profile retry bounds
|
||||
# ============================================================
|
||||
|
||||
Scenario: Action accepts safety profile with valid retry count
|
||||
Given a safety profile with max_retries_per_step 10
|
||||
When I create an action with the safety profile attached
|
||||
Then the action with safety profile should be created successfully
|
||||
And the action safety profile max_retries_per_step should be 10
|
||||
|
||||
Scenario: Action rejects safety profile with retry count above 100
|
||||
When I try to create an action with safety profile retries 101
|
||||
Then an action safety profile validation error should be raised
|
||||
|
||||
Scenario: Action rejects safety profile with negative retry count
|
||||
When I try to create an action with safety profile retries -1
|
||||
Then an action safety profile validation error should be raised
|
||||
|
||||
# ============================================================
|
||||
# Action with safety profile zero cost
|
||||
# ============================================================
|
||||
|
||||
Scenario: Action accepts safety profile with zero cost per plan
|
||||
Given a safety profile with max_cost_per_plan 0.0 and max_total_cost 100.0
|
||||
When I create an action with the safety profile attached
|
||||
Then the action with safety profile should be created successfully
|
||||
And the action safety profile max_cost_per_plan should be 0.0
|
||||
|
||||
# ============================================================
|
||||
# Action with equal cost boundaries (F-11)
|
||||
# ============================================================
|
||||
|
||||
Scenario: Action accepts safety profile with equal cost per plan and total
|
||||
Given a safety profile with max_cost_per_plan 100.0 and max_total_cost 100.0
|
||||
When I create an action with the safety profile attached
|
||||
Then the action with safety profile should be created successfully
|
||||
And the action safety profile max_cost_per_plan should be 100.0
|
||||
And the action safety profile max_total_cost should be 100.0
|
||||
|
||||
# ============================================================
|
||||
# Action without safety profile (default)
|
||||
# ============================================================
|
||||
|
||||
Scenario: Action without safety profile uses no safety constraints
|
||||
When I create an action without a safety profile
|
||||
Then the action with safety profile should be created successfully
|
||||
And the action safety profile should be None
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Step definitions for SafetyProfile cost/retry bounds on Action."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from pydantic import ValidationError
|
||||
|
||||
from cleveragents.domain.models.core.action import Action
|
||||
from cleveragents.domain.models.core.safety_profile import SafetyProfile
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_action(**overrides: Any) -> Action:
|
||||
"""Create an Action with sensible defaults, applying overrides."""
|
||||
from cleveragents.domain.models.core.plan import NamespacedName
|
||||
|
||||
defaults: dict[str, Any] = {
|
||||
"namespaced_name": NamespacedName(namespace="test", name="safety-action"),
|
||||
"description": "Test action for safety profile tests",
|
||||
"definition_of_done": "All tests pass",
|
||||
"strategy_actor": "test/strategy",
|
||||
"execution_actor": "test/execution",
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return Action(**defaults)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps (safety profile construction for action attachment)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a safety profile with max_cost_per_plan {cpp:g} and max_total_cost {tc:g}")
|
||||
def step_given_safety_profile_with_costs(
|
||||
context: Context, cpp: float, tc: float
|
||||
) -> None:
|
||||
"""Create a SafetyProfile with cost bounds for later use."""
|
||||
context.action_safety_profile = SafetyProfile(
|
||||
max_cost_per_plan=cpp, max_total_cost=tc
|
||||
)
|
||||
|
||||
|
||||
@given("a safety profile with max_retries_per_step {retries:d}")
|
||||
def step_given_safety_profile_with_retries(context: Context, retries: int) -> None:
|
||||
"""Create a SafetyProfile with retry bounds for later use."""
|
||||
context.action_safety_profile = SafetyProfile(max_retries_per_step=retries)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps (action creation with safety profile)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I create an action with the safety profile attached")
|
||||
def step_create_action_with_safety_profile(context: Context) -> None:
|
||||
"""Create an Action with the previously-built safety profile."""
|
||||
context.action_error = None
|
||||
context.action_model = None
|
||||
try:
|
||||
context.action_model = _make_action(
|
||||
safety_profile=context.action_safety_profile
|
||||
)
|
||||
except ValidationError as exc:
|
||||
context.action_error = exc
|
||||
|
||||
|
||||
@when(
|
||||
"I try to create an action with safety profile cost_per_plan {cpp:g}"
|
||||
" and total {tc:g}"
|
||||
)
|
||||
def step_try_create_action_invalid_costs(
|
||||
context: Context, cpp: float, tc: float
|
||||
) -> None:
|
||||
"""Try creating an action with invalid cost bounds."""
|
||||
context.action_error = None
|
||||
context.action_model = None
|
||||
try:
|
||||
sp = SafetyProfile(max_cost_per_plan=cpp, max_total_cost=tc)
|
||||
context.action_model = _make_action(safety_profile=sp)
|
||||
except ValidationError as exc:
|
||||
context.action_error = exc
|
||||
|
||||
|
||||
@when("I try to create an action with safety profile retries {retries:d}")
|
||||
def step_try_create_action_invalid_retries(context: Context, retries: int) -> None:
|
||||
"""Try creating an action with invalid retry bounds."""
|
||||
context.action_error = None
|
||||
context.action_model = None
|
||||
try:
|
||||
sp = SafetyProfile(max_retries_per_step=retries)
|
||||
context.action_model = _make_action(safety_profile=sp)
|
||||
except ValidationError as exc:
|
||||
context.action_error = exc
|
||||
|
||||
|
||||
@when("I create an action without a safety profile")
|
||||
def step_create_action_no_safety_profile(context: Context) -> None:
|
||||
"""Create an Action without a safety profile."""
|
||||
context.action_error = None
|
||||
context.action_model = None
|
||||
try:
|
||||
context.action_model = _make_action()
|
||||
except ValidationError as exc:
|
||||
context.action_error = exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the action with safety profile should be created successfully")
|
||||
def step_action_with_safety_profile_created(context: Context) -> None:
|
||||
"""Assert that the action was created successfully."""
|
||||
assert context.action_error is None, (
|
||||
f"Expected no error, got: {context.action_error}"
|
||||
)
|
||||
assert context.action_model is not None, "Expected an Action model, got None"
|
||||
|
||||
|
||||
@then("the action safety profile max_cost_per_plan should be {expected:g}")
|
||||
def step_check_action_safety_cost_per_plan(context: Context, expected: float) -> None:
|
||||
"""Assert action safety profile max_cost_per_plan."""
|
||||
sp = context.action_model.safety_profile
|
||||
assert sp is not None, "Expected safety_profile to be set"
|
||||
assert sp.max_cost_per_plan == expected, (
|
||||
f"Expected max_cost_per_plan {expected}, got {sp.max_cost_per_plan}"
|
||||
)
|
||||
|
||||
|
||||
@then("the action safety profile max_total_cost should be {expected:g}")
|
||||
def step_check_action_safety_total_cost(context: Context, expected: float) -> None:
|
||||
"""Assert action safety profile max_total_cost."""
|
||||
sp = context.action_model.safety_profile
|
||||
assert sp is not None, "Expected safety_profile to be set"
|
||||
assert sp.max_total_cost == expected, (
|
||||
f"Expected max_total_cost {expected}, got {sp.max_total_cost}"
|
||||
)
|
||||
|
||||
|
||||
@then("the action safety profile max_retries_per_step should be {expected:d}")
|
||||
def step_check_action_safety_retries(context: Context, expected: int) -> None:
|
||||
"""Assert action safety profile max_retries_per_step."""
|
||||
sp = context.action_model.safety_profile
|
||||
assert sp is not None, "Expected safety_profile to be set"
|
||||
assert sp.max_retries_per_step == expected, (
|
||||
f"Expected max_retries_per_step {expected}, got {sp.max_retries_per_step}"
|
||||
)
|
||||
|
||||
|
||||
@then("an action safety profile validation error should be raised")
|
||||
def step_action_safety_validation_error(context: Context) -> None:
|
||||
"""Assert a validation error was raised for the action."""
|
||||
assert context.action_error is not None, (
|
||||
"Expected a validation error but none was raised"
|
||||
)
|
||||
assert isinstance(context.action_error, ValidationError), (
|
||||
f"Expected ValidationError, got {type(context.action_error).__name__}: "
|
||||
f"{context.action_error}"
|
||||
)
|
||||
|
||||
|
||||
@then('the action safety profile error should mention "{text}"')
|
||||
def step_action_safety_error_mentions(context: Context, text: str) -> None:
|
||||
"""Assert the action error mentions expected text."""
|
||||
err_str = str(context.action_error)
|
||||
assert text.lower() in err_str.lower(), (
|
||||
f"Expected error to mention '{text}', got: {err_str}"
|
||||
)
|
||||
|
||||
|
||||
@then("the action safety profile should be None")
|
||||
def step_check_action_safety_profile_none(context: Context) -> None:
|
||||
"""Assert that the action has no safety profile."""
|
||||
assert context.action_model.safety_profile is None, (
|
||||
f"Expected safety_profile to be None, got {context.action_model.safety_profile}"
|
||||
)
|
||||
@@ -570,10 +570,22 @@ def step_check_cats_count(context: Context, expected: int) -> None:
|
||||
assert actual == expected, f"Expected {expected} categories, got {actual}"
|
||||
|
||||
|
||||
@then('the safety allowed_skill_categories should contain "{cat1}" and "{cat2}"')
|
||||
def step_check_cats_content(context: Context, cat1: str, cat2: str) -> None:
|
||||
"""Check allowed_skill_categories contains specific values."""
|
||||
cats = context.safety_model.allowed_skill_categories
|
||||
assert cat1 in cats, f"Expected '{cat1}' in categories {cats}"
|
||||
assert cat2 in cats, f"Expected '{cat2}' in categories {cats}"
|
||||
|
||||
|
||||
@then("a safety validation error should be raised")
|
||||
def step_check_safety_error(context: Context) -> None:
|
||||
"""Verify validation error was raised."""
|
||||
assert context.safety_error is not None, "Expected a validation error"
|
||||
assert isinstance(context.safety_error, ValidationError), (
|
||||
f"Expected ValidationError, got {type(context.safety_error).__name__}: "
|
||||
f"{context.safety_error}"
|
||||
)
|
||||
|
||||
|
||||
@then("a safety FileNotFoundError should be raised")
|
||||
@@ -652,3 +664,85 @@ def step_check_action_cli_dict_safety(context: Context) -> None:
|
||||
assert "safety_profile" in cli_dict, (
|
||||
f"Expected 'safety_profile' in cli dict, keys: {list(cli_dict)}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boolean flag toggles (#333)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I create a safety profile with require_sandbox {value}")
|
||||
def step_create_safety_require_sandbox(context: Context, value: str) -> None:
|
||||
"""Create a safety profile with a specific require_sandbox value."""
|
||||
context.safety_model = _make_safety(require_sandbox=value.lower() == "true")
|
||||
context.safety_error = None
|
||||
|
||||
|
||||
@when("I create a safety profile with require_checkpoints {value}")
|
||||
def step_create_safety_require_checkpoints(context: Context, value: str) -> None:
|
||||
"""Create a safety profile with a specific require_checkpoints value."""
|
||||
context.safety_model = _make_safety(require_checkpoints=value.lower() == "true")
|
||||
context.safety_error = None
|
||||
|
||||
|
||||
@when("I create a safety profile with allow_unsafe_tools {value}")
|
||||
def step_create_safety_allow_unsafe(context: Context, value: str) -> None:
|
||||
"""Create a safety profile with a specific allow_unsafe_tools value."""
|
||||
context.safety_model = _make_safety(allow_unsafe_tools=value.lower() == "true")
|
||||
context.safety_error = None
|
||||
|
||||
|
||||
@when("I create a safety profile with require_human_approval {value}")
|
||||
def step_create_safety_require_human(context: Context, value: str) -> None:
|
||||
"""Create a safety profile with a specific require_human_approval value."""
|
||||
context.safety_model = _make_safety(require_human_approval=value.lower() == "true")
|
||||
context.safety_error = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cost per plan / total cost without the other (#333)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I create a safety profile with max_total_cost {value:g}")
|
||||
def step_create_safety_total_cost(context: Context, value: float) -> None:
|
||||
"""Create a safety profile with a specific max_total_cost."""
|
||||
context.safety_model = _make_safety(max_total_cost=value)
|
||||
context.safety_error = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type validation (#333)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I try to create a safety profile with string cost")
|
||||
def step_try_safety_string_cost(context: Context) -> None:
|
||||
"""Try creating a safety profile with cost as string (should fail)."""
|
||||
context.safety_error = None
|
||||
context.safety_model = None
|
||||
try:
|
||||
context.safety_model = _make_safety(max_cost_per_plan="not_a_number")
|
||||
except ValidationError as e:
|
||||
context.safety_error = e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Restrictive full integration (#333)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I create a restrictive safety profile")
|
||||
def step_create_restrictive_safety(context: Context) -> None:
|
||||
"""Create a restrictive safety profile with all constraints set."""
|
||||
context.safety_model = _make_safety(
|
||||
require_sandbox=True,
|
||||
require_checkpoints=True,
|
||||
require_human_approval=True,
|
||||
allow_unsafe_tools=False,
|
||||
max_cost_per_plan=10.0,
|
||||
max_total_cost=100.0,
|
||||
max_retries_per_step=1,
|
||||
allowed_skill_categories=["code", "test"],
|
||||
)
|
||||
context.safety_error = None
|
||||
|
||||
@@ -151,6 +151,82 @@ def _test_summary() -> None:
|
||||
print("summary-ok")
|
||||
|
||||
|
||||
def _test_validation() -> None:
|
||||
"""Verify constraint validation rejects invalid inputs."""
|
||||
from pydantic import ValidationError
|
||||
|
||||
# Negative cost rejected
|
||||
try:
|
||||
SafetyProfile(max_cost_per_plan=-1.0)
|
||||
print("ERROR: should reject negative cost", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except ValidationError:
|
||||
pass
|
||||
|
||||
# Retries above 100 rejected
|
||||
try:
|
||||
SafetyProfile(max_retries_per_step=101)
|
||||
print("ERROR: should reject retries > 100", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except ValidationError:
|
||||
pass
|
||||
|
||||
# Cross-field: cost_per_plan > total_cost rejected
|
||||
try:
|
||||
SafetyProfile(max_cost_per_plan=200.0, max_total_cost=100.0)
|
||||
print("ERROR: should reject cost > total", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except ValidationError:
|
||||
pass
|
||||
|
||||
# Valid boundary values accepted
|
||||
p = SafetyProfile(max_retries_per_step=0, max_cost_per_plan=0.0)
|
||||
assert p.max_retries_per_step == 0
|
||||
assert p.max_cost_per_plan == 0.0
|
||||
|
||||
print("safety-validation-ok")
|
||||
|
||||
|
||||
def _test_action_attach() -> None:
|
||||
"""Verify safety profile attaches to Action model."""
|
||||
from cleveragents.domain.models.core.action import Action
|
||||
|
||||
config = {
|
||||
"name": "local/robot-safety-test",
|
||||
"description": "Robot test for safety profile attachment",
|
||||
"definition_of_done": "Tests pass",
|
||||
"strategy_actor": "local/default",
|
||||
"execution_actor": "local/default",
|
||||
"safety_profile": {
|
||||
"require_sandbox": False,
|
||||
"max_cost_per_plan": 50.0,
|
||||
"max_retries_per_step": 2,
|
||||
},
|
||||
}
|
||||
action = Action.from_config(config)
|
||||
assert action.safety_profile is not None
|
||||
assert action.safety_profile.require_sandbox is False
|
||||
assert action.safety_profile.max_cost_per_plan == 50.0
|
||||
assert action.safety_profile.max_retries_per_step == 2
|
||||
|
||||
# Verify as_cli_dict includes safety_profile
|
||||
cli = action.as_cli_dict()
|
||||
assert "safety_profile" in cli
|
||||
|
||||
# Verify action without safety profile
|
||||
config_no_sp = {
|
||||
"name": "local/robot-no-safety",
|
||||
"description": "No safety profile",
|
||||
"definition_of_done": "Pass",
|
||||
"strategy_actor": "local/default",
|
||||
"execution_actor": "local/default",
|
||||
}
|
||||
action_no = Action.from_config(config_no_sp)
|
||||
assert action_no.safety_profile is None
|
||||
|
||||
print("safety-action-attach-ok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cmd = sys.argv[1] if len(sys.argv) > 1 else "summary"
|
||||
dispatch: dict[str, Any] = {
|
||||
@@ -159,6 +235,8 @@ if __name__ == "__main__":
|
||||
"from-yaml": _test_from_yaml,
|
||||
"serialize": _test_serialize,
|
||||
"ref": _test_ref,
|
||||
"validation": _test_validation,
|
||||
"action-attach": _test_action_attach,
|
||||
"summary": _test_summary,
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,18 @@ Safety Profile Ref Creation
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} safety-ref-ok
|
||||
|
||||
Safety Profile Validation Rules
|
||||
[Documentation] Verify constraint validation rejects invalid inputs
|
||||
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} validation cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} safety-validation-ok
|
||||
|
||||
Safety Profile Action Attachment
|
||||
[Documentation] Verify safety profile attaches to Action model
|
||||
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} action-attach cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} safety-action-attach-ok
|
||||
|
||||
Safety Profile Summary
|
||||
[Documentation] Print summary of safety profile capabilities
|
||||
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} summary cwd=${WORKSPACE}
|
||||
|
||||
Reference in New Issue
Block a user