diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ee2b079..5abc2e140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,13 @@ method. CLI `plan status` shows per-project changeset summaries for multi-project plans. Includes Behave BDD scenarios, Robot Framework smoke tests, ASV benchmarks, and reference documentation. (#199) +- Added safety profile enforcement test coverage: 12 Behave BDD scenarios for boolean flag + toggles, deny-none semantics, cost-without-total, type validation, negative-cost rejection, + upper-bound retries, and restrictive full-constraint profiles; 8 cost/retry + Action-integration scenarios including equal-value boundary; 2 Robot Framework + smoke tests (validation rules, action attachment); and ASV benchmarks for test-scenario + runtime baselines. Updated `docs/development/testing.md` with safety profile test fixture + documentation. (#333) - Added large-project hierarchical decomposition with 4+ levels and bounded context per subplan. Includes clustering heuristics (directory, language, size), bounded dependency closure with memoization for 10K+ files, DAG execution ordering with cycle detection, diff --git a/benchmarks/safety_profile_tests_bench.py b/benchmarks/safety_profile_tests_bench.py new file mode 100644 index 000000000..c3e833dd9 --- /dev/null +++ b/benchmarks/safety_profile_tests_bench.py @@ -0,0 +1,166 @@ +"""ASV benchmarks for SafetyProfile domain model scenario runtime. + +Measures: +- SafetyProfile construction time (defaults and explicit fields) +- Cross-field cost validation overhead +- Skill category deduplication overhead +- SafetyProfile serialization / deserialization round-trip +- Action with safety profile attachment construction +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +try: + from cleveragents.domain.models.core.safety_profile import SafetyProfile +except ModuleNotFoundError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cleveragents.domain.models.core.safety_profile import SafetyProfile + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_safety_profile() -> SafetyProfile: + """Create a fully-populated SafetyProfile.""" + return SafetyProfile( + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + require_human_approval=True, + allowed_skill_categories=["code", "data", "infra"], + max_cost_per_plan=50.0, + max_total_cost=200.0, + max_retries_per_step=5, + ) + + +# --------------------------------------------------------------------------- +# Construction benchmarks +# --------------------------------------------------------------------------- + + +class SafetyProfileConstructionSuite: + """Benchmark SafetyProfile construction variants.""" + + def time_default_construction(self) -> None: + """Measure default SafetyProfile construction time.""" + SafetyProfile() + + def time_full_construction(self) -> None: + """Measure fully-populated SafetyProfile construction time.""" + _make_safety_profile() + + def time_minimal_construction_overrides(self) -> None: + """Measure SafetyProfile with minimal overrides.""" + SafetyProfile(require_sandbox=False, allow_unsafe_tools=True) + + +# --------------------------------------------------------------------------- +# Validation benchmarks +# --------------------------------------------------------------------------- + + +class SafetyProfileValidationSuite: + """Benchmark SafetyProfile validation operations.""" + + def time_cost_bounds_validation_valid(self) -> None: + """Measure valid cost bounds validation.""" + SafetyProfile(max_cost_per_plan=50.0, max_total_cost=100.0) + + def time_cost_bounds_validation_equal(self) -> None: + """Measure cost bounds validation with equal values.""" + SafetyProfile(max_cost_per_plan=100.0, max_total_cost=100.0) + + def time_retry_bounds_validation(self) -> None: + """Measure retry bounds validation.""" + SafetyProfile(max_retries_per_step=50) + + def time_skill_category_deduplication(self) -> None: + """Measure skill category trimming and deduplication.""" + SafetyProfile( + allowed_skill_categories=[ + "code", + " code ", + "data", + "code", + "infra", + " data ", + ] + ) + + +# --------------------------------------------------------------------------- +# Serialization benchmarks +# --------------------------------------------------------------------------- + + +class SafetyProfileSerializationSuite: + """Benchmark SafetyProfile serialization round-trips.""" + + def setup(self) -> None: + """Create a profile for serialization benchmarks.""" + self.profile = _make_safety_profile() + self.profile_dict = self.profile.model_dump() + self.profile_json = self.profile.model_dump_json() + + def time_model_dump(self) -> None: + """Measure SafetyProfile to dict serialization.""" + self.profile.model_dump() + + def time_model_dump_json(self) -> None: + """Measure SafetyProfile to JSON serialization.""" + self.profile.model_dump_json() + + def time_model_validate_from_dict(self) -> None: + """Measure SafetyProfile from dict deserialization.""" + SafetyProfile.model_validate(self.profile_dict) + + def time_model_validate_json(self) -> None: + """Measure SafetyProfile from JSON deserialization.""" + SafetyProfile.model_validate_json(self.profile_json) + + +# --------------------------------------------------------------------------- +# Action attachment benchmarks +# --------------------------------------------------------------------------- + + +class SafetyProfileActionAttachmentSuite: + """Benchmark Action construction with SafetyProfile attachment.""" + + def setup(self) -> None: + """Pre-build a safety profile and import Action/NamespacedName.""" + from cleveragents.domain.models.core.action import Action + from cleveragents.domain.models.core.plan import NamespacedName + + self.Action = Action + self.NamespacedName = NamespacedName + self.safety_profile = _make_safety_profile() + + def time_action_with_safety_profile(self) -> None: + """Measure Action construction with safety profile attached.""" + nn = self.NamespacedName(namespace="bench", name="safety-action") + self.Action( + namespaced_name=nn, + description="Benchmark action with safety profile", + definition_of_done="Benchmark complete", + strategy_actor="bench/strategy", + execution_actor="bench/execution", + safety_profile=self.safety_profile, + ) + + def time_action_without_safety_profile(self) -> None: + """Measure Action construction without safety profile.""" + nn = self.NamespacedName(namespace="bench", name="no-safety") + self.Action( + namespaced_name=nn, + description="Benchmark action without safety profile", + definition_of_done="Benchmark complete", + strategy_actor="bench/strategy", + execution_actor="bench/execution", + ) diff --git a/docs/development/testing.md b/docs/development/testing.md index a76ac8ea7..8f487df5d 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -1507,4 +1507,51 @@ nox --- +## Safety Profile Test Fixtures + +### Behave Feature Files + +- **`features/safety_profile.feature`** — Core model validation: defaults, + constraint validation, cross-field cost bounds, category sanitization, + immutability, `from_config`/`from_yaml` factories, `SafetyProfileRef`, + `resolve_safety_profile` stub, `model_dump` round-trip, Action integration, + boolean flag toggles, and restrictive full-constraint profiles. +- **`features/safety_profile_cost_retry.feature`** — Action-level cost/retry + bounds: attaching `SafetyProfile` to `Action`, rejecting cross-field + violations through action creation, valid/zero boundaries, and absent + safety profile semantics. + +### YAML Config Example + +```yaml +# Example safety profile YAML for test fixtures +require_sandbox: true +require_checkpoints: true +require_human_approval: false +allow_unsafe_tools: false +max_cost_per_plan: 50.0 +max_total_cost: 200.0 +max_retries_per_step: 5 +allowed_skill_categories: + - code + - test + - deploy +``` + +### Robot Framework + +- **`robot/safety_profile.robot`** — Smoke tests: default profile, config + loading, YAML loading, serialization, `SafetyProfileRef`, constraint + validation, Action attachment, and summary. + +### ASV Benchmarks + +- **`benchmarks/safety_profile_model_bench.py`** — Model construction, + validation overhead, `from_yaml`, serialization, resolution stub. +- **`benchmarks/safety_profile_tests_bench.py`** — Test-scenario runtime: + construction variants, cost/retry validation, category deduplication, + serialization round-trips, Action attachment overhead. + +--- + ## diff --git a/features/safety_profile.feature b/features/safety_profile.feature index 3729cc894..2d239816f 100644 --- a/features/safety_profile.feature +++ b/features/safety_profile.feature @@ -226,3 +226,80 @@ Feature: Safety Profile Domain Model Scenario: Safety profile from_yaml rejects non-dict content When I try to load a safety profile from yaml with non-dict content Then a safety ValueError should be raised + + # ---- Boolean flag toggles (#333) ---- + + Scenario: Safety profile accepts sandbox disabled + When I create a safety profile with require_sandbox false + Then the safety profile should be created + And the safety require_sandbox should be false + + Scenario: Safety profile accepts checkpoints disabled + When I create a safety profile with require_checkpoints false + Then the safety profile should be created + And the safety require_checkpoints should be false + + Scenario: Safety profile accepts unsafe tools enabled + When I create a safety profile with allow_unsafe_tools true + Then the safety profile should be created + And the safety allow_unsafe_tools should be true + + Scenario: Safety profile accepts human approval required + When I create a safety profile with require_human_approval true + Then the safety profile should be created + And the safety require_human_approval should be true + + # ---- Deny-none semantics (#333) ---- + + Scenario: Empty skill categories list means all categories allowed + When I create a default safety profile + Then the safety profile should be created + And the safety allowed_skill_categories should be empty + + # ---- Cost per plan without total cost (#333) ---- + + Scenario: Safety profile allows cost per plan without total cost + When I create a safety profile with max_cost_per_plan 50.0 + Then the safety profile should be created + And the safety max_cost_per_plan should be 50.0 + And the safety max_total_cost should be none + + Scenario: Safety profile allows total cost without cost per plan + When I create a safety profile with max_total_cost 200.0 + Then the safety profile should be created + And the safety max_total_cost should be 200.0 + And the safety max_cost_per_plan should be none + + # ---- Type validation (#333) ---- + + Scenario: Safety profile rejects cost per plan as string + When I try to create a safety profile with string cost + Then a safety validation error should be raised + + # ---- Negative cost rejection (#333/F-07) ---- + + Scenario: Safety profile rejects negative cost per plan via profile + When I try to create a safety profile with max_cost_per_plan -5.0 + Then a safety validation error should be raised + + # ---- Boundary value tests for max_retries (#333/F-09) ---- + + Scenario: Safety profile accepts max_retries_per_step at upper bound 100 + When I create a safety profile with max_retries_per_step 100 + Then the safety profile should be created + And the safety max_retries_per_step should be 100 + + # ---- Restrictive full integration (#333) ---- + + Scenario: Restrictive safety profile validates all constraints + When I create a restrictive safety profile + Then the safety profile should be created + And the safety require_sandbox should be true + And the safety require_checkpoints should be true + And the safety require_human_approval should be true + And the safety allow_unsafe_tools should be false + And the safety max_cost_per_plan should be 10.0 + And the safety max_total_cost should be 100.0 + And the safety max_retries_per_step should be 1 + And the safety allowed_skill_categories count should be 2 + And the safety allowed_skill_categories should contain "code" and "test" diff --git a/features/safety_profile_cost_retry.feature b/features/safety_profile_cost_retry.feature new file mode 100644 index 000000000..6a3cfcfb2 --- /dev/null +++ b/features/safety_profile_cost_retry.feature @@ -0,0 +1,67 @@ +Feature: Safety Profile cost and retry bounds on action creation + Validates that cost and retry bounds on SafetyProfile are enforced + during action creation when a safety_profile is attached to an Action. + + # ============================================================ + # Action with safety profile cost bounds + # ============================================================ + + Scenario: Action accepts safety profile with valid cost bounds + Given a safety profile with max_cost_per_plan 50.0 and max_total_cost 200.0 + When I create an action with the safety profile attached + Then the action with safety profile should be created successfully + And the action safety profile max_cost_per_plan should be 50.0 + And the action safety profile max_total_cost should be 200.0 + + Scenario: Action rejects safety profile where cost per plan exceeds total + When I try to create an action with safety profile cost_per_plan 300.0 and total 100.0 + Then an action safety profile validation error should be raised + And the action safety profile error should mention "max_cost_per_plan" + + # ============================================================ + # Action with safety profile retry bounds + # ============================================================ + + Scenario: Action accepts safety profile with valid retry count + Given a safety profile with max_retries_per_step 10 + When I create an action with the safety profile attached + Then the action with safety profile should be created successfully + And the action safety profile max_retries_per_step should be 10 + + Scenario: Action rejects safety profile with retry count above 100 + When I try to create an action with safety profile retries 101 + Then an action safety profile validation error should be raised + + Scenario: Action rejects safety profile with negative retry count + When I try to create an action with safety profile retries -1 + Then an action safety profile validation error should be raised + + # ============================================================ + # Action with safety profile zero cost + # ============================================================ + + Scenario: Action accepts safety profile with zero cost per plan + Given a safety profile with max_cost_per_plan 0.0 and max_total_cost 100.0 + When I create an action with the safety profile attached + Then the action with safety profile should be created successfully + And the action safety profile max_cost_per_plan should be 0.0 + + # ============================================================ + # Action with equal cost boundaries (F-11) + # ============================================================ + + Scenario: Action accepts safety profile with equal cost per plan and total + Given a safety profile with max_cost_per_plan 100.0 and max_total_cost 100.0 + When I create an action with the safety profile attached + Then the action with safety profile should be created successfully + And the action safety profile max_cost_per_plan should be 100.0 + And the action safety profile max_total_cost should be 100.0 + + # ============================================================ + # Action without safety profile (default) + # ============================================================ + + Scenario: Action without safety profile uses no safety constraints + When I create an action without a safety profile + Then the action with safety profile should be created successfully + And the action safety profile should be None diff --git a/features/steps/safety_profile_cost_retry_steps.py b/features/steps/safety_profile_cost_retry_steps.py new file mode 100644 index 000000000..efe6fa0e3 --- /dev/null +++ b/features/steps/safety_profile_cost_retry_steps.py @@ -0,0 +1,183 @@ +"""Step definitions for SafetyProfile cost/retry bounds on Action.""" + +from __future__ import annotations + +from typing import Any + +from behave import given, then, when +from behave.runner import Context +from pydantic import ValidationError + +from cleveragents.domain.models.core.action import Action +from cleveragents.domain.models.core.safety_profile import SafetyProfile + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_action(**overrides: Any) -> Action: + """Create an Action with sensible defaults, applying overrides.""" + from cleveragents.domain.models.core.plan import NamespacedName + + defaults: dict[str, Any] = { + "namespaced_name": NamespacedName(namespace="test", name="safety-action"), + "description": "Test action for safety profile tests", + "definition_of_done": "All tests pass", + "strategy_actor": "test/strategy", + "execution_actor": "test/execution", + } + defaults.update(overrides) + return Action(**defaults) + + +# --------------------------------------------------------------------------- +# Given steps (safety profile construction for action attachment) +# --------------------------------------------------------------------------- + + +@given("a safety profile with max_cost_per_plan {cpp:g} and max_total_cost {tc:g}") +def step_given_safety_profile_with_costs( + context: Context, cpp: float, tc: float +) -> None: + """Create a SafetyProfile with cost bounds for later use.""" + context.action_safety_profile = SafetyProfile( + max_cost_per_plan=cpp, max_total_cost=tc + ) + + +@given("a safety profile with max_retries_per_step {retries:d}") +def step_given_safety_profile_with_retries(context: Context, retries: int) -> None: + """Create a SafetyProfile with retry bounds for later use.""" + context.action_safety_profile = SafetyProfile(max_retries_per_step=retries) + + +# --------------------------------------------------------------------------- +# When steps (action creation with safety profile) +# --------------------------------------------------------------------------- + + +@when("I create an action with the safety profile attached") +def step_create_action_with_safety_profile(context: Context) -> None: + """Create an Action with the previously-built safety profile.""" + context.action_error = None + context.action_model = None + try: + context.action_model = _make_action( + safety_profile=context.action_safety_profile + ) + except ValidationError as exc: + context.action_error = exc + + +@when( + "I try to create an action with safety profile cost_per_plan {cpp:g}" + " and total {tc:g}" +) +def step_try_create_action_invalid_costs( + context: Context, cpp: float, tc: float +) -> None: + """Try creating an action with invalid cost bounds.""" + context.action_error = None + context.action_model = None + try: + sp = SafetyProfile(max_cost_per_plan=cpp, max_total_cost=tc) + context.action_model = _make_action(safety_profile=sp) + except ValidationError as exc: + context.action_error = exc + + +@when("I try to create an action with safety profile retries {retries:d}") +def step_try_create_action_invalid_retries(context: Context, retries: int) -> None: + """Try creating an action with invalid retry bounds.""" + context.action_error = None + context.action_model = None + try: + sp = SafetyProfile(max_retries_per_step=retries) + context.action_model = _make_action(safety_profile=sp) + except ValidationError as exc: + context.action_error = exc + + +@when("I create an action without a safety profile") +def step_create_action_no_safety_profile(context: Context) -> None: + """Create an Action without a safety profile.""" + context.action_error = None + context.action_model = None + try: + context.action_model = _make_action() + except ValidationError as exc: + context.action_error = exc + + +# --------------------------------------------------------------------------- +# Assertions +# --------------------------------------------------------------------------- + + +@then("the action with safety profile should be created successfully") +def step_action_with_safety_profile_created(context: Context) -> None: + """Assert that the action was created successfully.""" + assert context.action_error is None, ( + f"Expected no error, got: {context.action_error}" + ) + assert context.action_model is not None, "Expected an Action model, got None" + + +@then("the action safety profile max_cost_per_plan should be {expected:g}") +def step_check_action_safety_cost_per_plan(context: Context, expected: float) -> None: + """Assert action safety profile max_cost_per_plan.""" + sp = context.action_model.safety_profile + assert sp is not None, "Expected safety_profile to be set" + assert sp.max_cost_per_plan == expected, ( + f"Expected max_cost_per_plan {expected}, got {sp.max_cost_per_plan}" + ) + + +@then("the action safety profile max_total_cost should be {expected:g}") +def step_check_action_safety_total_cost(context: Context, expected: float) -> None: + """Assert action safety profile max_total_cost.""" + sp = context.action_model.safety_profile + assert sp is not None, "Expected safety_profile to be set" + assert sp.max_total_cost == expected, ( + f"Expected max_total_cost {expected}, got {sp.max_total_cost}" + ) + + +@then("the action safety profile max_retries_per_step should be {expected:d}") +def step_check_action_safety_retries(context: Context, expected: int) -> None: + """Assert action safety profile max_retries_per_step.""" + sp = context.action_model.safety_profile + assert sp is not None, "Expected safety_profile to be set" + assert sp.max_retries_per_step == expected, ( + f"Expected max_retries_per_step {expected}, got {sp.max_retries_per_step}" + ) + + +@then("an action safety profile validation error should be raised") +def step_action_safety_validation_error(context: Context) -> None: + """Assert a validation error was raised for the action.""" + assert context.action_error is not None, ( + "Expected a validation error but none was raised" + ) + assert isinstance(context.action_error, ValidationError), ( + f"Expected ValidationError, got {type(context.action_error).__name__}: " + f"{context.action_error}" + ) + + +@then('the action safety profile error should mention "{text}"') +def step_action_safety_error_mentions(context: Context, text: str) -> None: + """Assert the action error mentions expected text.""" + err_str = str(context.action_error) + assert text.lower() in err_str.lower(), ( + f"Expected error to mention '{text}', got: {err_str}" + ) + + +@then("the action safety profile should be None") +def step_check_action_safety_profile_none(context: Context) -> None: + """Assert that the action has no safety profile.""" + assert context.action_model.safety_profile is None, ( + f"Expected safety_profile to be None, got {context.action_model.safety_profile}" + ) diff --git a/features/steps/safety_profile_steps.py b/features/steps/safety_profile_steps.py index 67f3be37f..526f03c8e 100644 --- a/features/steps/safety_profile_steps.py +++ b/features/steps/safety_profile_steps.py @@ -570,10 +570,22 @@ def step_check_cats_count(context: Context, expected: int) -> None: assert actual == expected, f"Expected {expected} categories, got {actual}" +@then('the safety allowed_skill_categories should contain "{cat1}" and "{cat2}"') +def step_check_cats_content(context: Context, cat1: str, cat2: str) -> None: + """Check allowed_skill_categories contains specific values.""" + cats = context.safety_model.allowed_skill_categories + assert cat1 in cats, f"Expected '{cat1}' in categories {cats}" + assert cat2 in cats, f"Expected '{cat2}' in categories {cats}" + + @then("a safety validation error should be raised") def step_check_safety_error(context: Context) -> None: """Verify validation error was raised.""" assert context.safety_error is not None, "Expected a validation error" + assert isinstance(context.safety_error, ValidationError), ( + f"Expected ValidationError, got {type(context.safety_error).__name__}: " + f"{context.safety_error}" + ) @then("a safety FileNotFoundError should be raised") @@ -652,3 +664,85 @@ def step_check_action_cli_dict_safety(context: Context) -> None: assert "safety_profile" in cli_dict, ( f"Expected 'safety_profile' in cli dict, keys: {list(cli_dict)}" ) + + +# --------------------------------------------------------------------------- +# Boolean flag toggles (#333) +# --------------------------------------------------------------------------- + + +@when("I create a safety profile with require_sandbox {value}") +def step_create_safety_require_sandbox(context: Context, value: str) -> None: + """Create a safety profile with a specific require_sandbox value.""" + context.safety_model = _make_safety(require_sandbox=value.lower() == "true") + context.safety_error = None + + +@when("I create a safety profile with require_checkpoints {value}") +def step_create_safety_require_checkpoints(context: Context, value: str) -> None: + """Create a safety profile with a specific require_checkpoints value.""" + context.safety_model = _make_safety(require_checkpoints=value.lower() == "true") + context.safety_error = None + + +@when("I create a safety profile with allow_unsafe_tools {value}") +def step_create_safety_allow_unsafe(context: Context, value: str) -> None: + """Create a safety profile with a specific allow_unsafe_tools value.""" + context.safety_model = _make_safety(allow_unsafe_tools=value.lower() == "true") + context.safety_error = None + + +@when("I create a safety profile with require_human_approval {value}") +def step_create_safety_require_human(context: Context, value: str) -> None: + """Create a safety profile with a specific require_human_approval value.""" + context.safety_model = _make_safety(require_human_approval=value.lower() == "true") + context.safety_error = None + + +# --------------------------------------------------------------------------- +# Cost per plan / total cost without the other (#333) +# --------------------------------------------------------------------------- + + +@when("I create a safety profile with max_total_cost {value:g}") +def step_create_safety_total_cost(context: Context, value: float) -> None: + """Create a safety profile with a specific max_total_cost.""" + context.safety_model = _make_safety(max_total_cost=value) + context.safety_error = None + + +# --------------------------------------------------------------------------- +# Type validation (#333) +# --------------------------------------------------------------------------- + + +@when("I try to create a safety profile with string cost") +def step_try_safety_string_cost(context: Context) -> None: + """Try creating a safety profile with cost as string (should fail).""" + context.safety_error = None + context.safety_model = None + try: + context.safety_model = _make_safety(max_cost_per_plan="not_a_number") + except ValidationError as e: + context.safety_error = e + + +# --------------------------------------------------------------------------- +# Restrictive full integration (#333) +# --------------------------------------------------------------------------- + + +@when("I create a restrictive safety profile") +def step_create_restrictive_safety(context: Context) -> None: + """Create a restrictive safety profile with all constraints set.""" + context.safety_model = _make_safety( + require_sandbox=True, + require_checkpoints=True, + require_human_approval=True, + allow_unsafe_tools=False, + max_cost_per_plan=10.0, + max_total_cost=100.0, + max_retries_per_step=1, + allowed_skill_categories=["code", "test"], + ) + context.safety_error = None diff --git a/robot/helper_safety_profile.py b/robot/helper_safety_profile.py index ec1efc6ff..6312b830d 100644 --- a/robot/helper_safety_profile.py +++ b/robot/helper_safety_profile.py @@ -151,6 +151,82 @@ def _test_summary() -> None: print("summary-ok") +def _test_validation() -> None: + """Verify constraint validation rejects invalid inputs.""" + from pydantic import ValidationError + + # Negative cost rejected + try: + SafetyProfile(max_cost_per_plan=-1.0) + print("ERROR: should reject negative cost", file=sys.stderr) + sys.exit(1) + except ValidationError: + pass + + # Retries above 100 rejected + try: + SafetyProfile(max_retries_per_step=101) + print("ERROR: should reject retries > 100", file=sys.stderr) + sys.exit(1) + except ValidationError: + pass + + # Cross-field: cost_per_plan > total_cost rejected + try: + SafetyProfile(max_cost_per_plan=200.0, max_total_cost=100.0) + print("ERROR: should reject cost > total", file=sys.stderr) + sys.exit(1) + except ValidationError: + pass + + # Valid boundary values accepted + p = SafetyProfile(max_retries_per_step=0, max_cost_per_plan=0.0) + assert p.max_retries_per_step == 0 + assert p.max_cost_per_plan == 0.0 + + print("safety-validation-ok") + + +def _test_action_attach() -> None: + """Verify safety profile attaches to Action model.""" + from cleveragents.domain.models.core.action import Action + + config = { + "name": "local/robot-safety-test", + "description": "Robot test for safety profile attachment", + "definition_of_done": "Tests pass", + "strategy_actor": "local/default", + "execution_actor": "local/default", + "safety_profile": { + "require_sandbox": False, + "max_cost_per_plan": 50.0, + "max_retries_per_step": 2, + }, + } + action = Action.from_config(config) + assert action.safety_profile is not None + assert action.safety_profile.require_sandbox is False + assert action.safety_profile.max_cost_per_plan == 50.0 + assert action.safety_profile.max_retries_per_step == 2 + + # Verify as_cli_dict includes safety_profile + cli = action.as_cli_dict() + assert "safety_profile" in cli + + # Verify action without safety profile + config_no_sp = { + "name": "local/robot-no-safety", + "description": "No safety profile", + "definition_of_done": "Pass", + "strategy_actor": "local/default", + "execution_actor": "local/default", + } + action_no = Action.from_config(config_no_sp) + assert action_no.safety_profile is None + + print("safety-action-attach-ok") + + if __name__ == "__main__": cmd = sys.argv[1] if len(sys.argv) > 1 else "summary" dispatch: dict[str, Any] = { @@ -159,6 +235,8 @@ if __name__ == "__main__": "from-yaml": _test_from_yaml, "serialize": _test_serialize, "ref": _test_ref, + "validation": _test_validation, + "action-attach": _test_action_attach, "summary": _test_summary, } diff --git a/robot/safety_profile.robot b/robot/safety_profile.robot index d961a7ac8..5865158c3 100644 --- a/robot/safety_profile.robot +++ b/robot/safety_profile.robot @@ -38,6 +38,18 @@ Safety Profile Ref Creation Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} safety-ref-ok +Safety Profile Validation Rules + [Documentation] Verify constraint validation rejects invalid inputs + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} validation cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} safety-validation-ok + +Safety Profile Action Attachment + [Documentation] Verify safety profile attaches to Action model + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} action-attach cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} safety-action-attach-ok + Safety Profile Summary [Documentation] Print summary of safety profile capabilities ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} summary cwd=${WORKSPACE}