diff --git a/benchmarks/automation_profile_bench.py b/benchmarks/automation_profile_bench.py new file mode 100644 index 000000000..ff50ecae2 --- /dev/null +++ b/benchmarks/automation_profile_bench.py @@ -0,0 +1,126 @@ +"""ASV benchmarks for AutomationProfile validation and serialization. + +Measures the performance of: +- AutomationProfile model construction (Pydantic validation) +- AutomationProfile.model_dump() serialization +- AutomationProfile.from_config() factory +- Built-in profile lookup via get_builtin_profile() +- BUILTIN_PROFILES iteration +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +try: + from cleveragents.domain.models.core.automation_profile import ( + BUILTIN_PROFILES, + AutomationProfile, + get_builtin_profile, + ) +except ModuleNotFoundError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cleveragents.domain.models.core.automation_profile import ( + BUILTIN_PROFILES, + AutomationProfile, + get_builtin_profile, + ) + + +def _make_profile() -> AutomationProfile: + """Create a fully-populated profile for benchmarking.""" + return AutomationProfile( + name="bench/test-profile", + description="Benchmark profile", + auto_strategize=0.7, + auto_execute=0.5, + auto_apply=1.0, + auto_decisions_strategize=0.6, + auto_decisions_execute=0.8, + auto_validation_fix=0.3, + auto_strategy_revision=0.9, + auto_reversion_from_apply=0.4, + auto_child_plans=0.7, + auto_retry_transient=0.1, + auto_checkpoint_restore=0.5, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ) + + +class ProfileValidationSuite: + """Benchmark AutomationProfile construction.""" + + def time_profile_construction(self) -> None: + """Benchmark fully-populated profile creation.""" + _make_profile() + + def time_profile_minimal_construction(self) -> None: + """Benchmark minimal profile creation.""" + AutomationProfile(name="bench/minimal") + + def time_profile_all_thresholds_max(self) -> None: + """Benchmark profile with all thresholds at 1.0.""" + AutomationProfile( + name="bench/max", + auto_strategize=1.0, + auto_execute=1.0, + auto_apply=1.0, + auto_decisions_strategize=1.0, + auto_decisions_execute=1.0, + auto_validation_fix=1.0, + auto_strategy_revision=1.0, + auto_reversion_from_apply=1.0, + auto_child_plans=1.0, + auto_retry_transient=1.0, + auto_checkpoint_restore=1.0, + ) + + +class ProfileSerializationSuite: + """Benchmark AutomationProfile serialization.""" + + def setup(self) -> None: + """Create objects for serialization benchmarks.""" + self.profile = _make_profile() + + def time_profile_model_dump(self) -> None: + """Benchmark model_dump() serialization.""" + self.profile.model_dump() + + def time_profile_model_dump_json(self) -> None: + """Benchmark model_dump_json() JSON serialization.""" + self.profile.model_dump_json() + + +class ProfileFromConfigSuite: + """Benchmark AutomationProfile.from_config() factory.""" + + def setup(self) -> None: + """Prepare config dicts for benchmarks.""" + self.config = { + "name": "bench/from-config", + "description": "Config benchmark", + "auto_strategize": 0.7, + "auto_execute": 0.5, + "auto_apply": 1.0, + } + + def time_profile_from_config(self) -> None: + """Benchmark from_config().""" + AutomationProfile.from_config(self.config) + + +class BuiltinProfileSuite: + """Benchmark built-in profile operations.""" + + def time_get_builtin_profile(self) -> None: + """Benchmark get_builtin_profile() lookup.""" + get_builtin_profile("cautious") + + def time_iterate_all_builtins(self) -> None: + """Benchmark iterating all built-in profiles.""" + for name in BUILTIN_PROFILES: + _ = BUILTIN_PROFILES[name] diff --git a/docs/reference/automation_profiles.md b/docs/reference/automation_profiles.md new file mode 100644 index 000000000..b2505be9d --- /dev/null +++ b/docs/reference/automation_profiles.md @@ -0,0 +1,110 @@ +# Automation Profiles + +Automation Profiles control how much autonomy the CleverAgents system has at each phase of plan execution. Each profile defines a set of **threshold values** and **safety requirements** that determine when the system may proceed automatically versus when it must wait for human approval. + +## Threshold Semantics + +Each threshold field is a float in the range `[0.0, 1.0]`: + +| Value | Meaning | +|-------|---------| +| `0.0` | Fully automatic — no human gate required | +| `1.0` | Always requires human approval | +| `0.0 < v < 1.0` | Probabilistic — the system may proceed if its confidence exceeds the threshold | + +### Threshold Fields + +| Field | Category | Description | +|-------|----------|-------------| +| `auto_strategize` | Phase transition | Gate before entering the strategy phase | +| `auto_execute` | Phase transition | Gate before entering the execution phase | +| `auto_apply` | Phase transition | Gate before applying changes | +| `auto_decisions_strategize` | Decision autonomy | Gate for decisions during strategy | +| `auto_decisions_execute` | Decision autonomy | Gate for decisions during execution | +| `auto_validation_fix` | Self-repair | Gate for automatic validation fixes | +| `auto_strategy_revision` | Self-repair | Gate for automatic strategy revision | +| `auto_reversion_from_apply` | Self-repair | Gate for reverting from the apply phase | +| `auto_child_plans` | Child plans | Gate for spawning child plans | +| `auto_retry_transient` | Retry | Gate for retrying transient failures | +| `auto_checkpoint_restore` | Checkpoint | Gate for automatic checkpoint restoration | + +### Safety Fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `require_sandbox` | bool | `true` | Execution must happen in a sandbox | +| `require_checkpoints` | bool | `true` | Checkpoints must be created before writes | +| `allow_unsafe_tools` | bool | `false` | Tools flagged as `unsafe` may be invoked | + +## Built-in Profiles + +Eight profiles ship with every CleverAgents installation: + +| Flag | manual | review | supervised | cautious | trusted | auto | ci | full-auto | +|------|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| auto_strategize | 1.0 | 0.0 | 0.0 | 0.7 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_execute | 1.0 | 0.0 | 1.0 | 0.7 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_apply | 1.0 | 1.0 | 1.0 | 1.0 | 1.0 | 1.0 | 0.0 | 0.0 | +| auto_decisions_strategize | 1.0 | 1.0 | 0.0 | 0.6 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_decisions_execute | 1.0 | 1.0 | 1.0 | 0.8 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_validation_fix | 1.0 | 1.0 | 1.0 | 0.7 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_strategy_revision | 1.0 | 1.0 | 1.0 | 0.8 | 1.0 | 0.0 | 0.0 | 0.0 | +| auto_reversion_from_apply | 1.0 | 1.0 | 1.0 | 0.9 | 1.0 | 1.0 | 0.0 | 0.0 | +| auto_child_plans | 1.0 | 0.0 | 1.0 | 0.7 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_retry_transient | 1.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | 0.0 | +| auto_checkpoint_restore | 1.0 | 1.0 | 1.0 | 0.6 | 1.0 | 0.0 | 0.0 | 0.0 | +| require_sandbox | true | true | true | true | true | true | true | false | +| require_checkpoints | true | true | true | true | true | true | true | false | +| allow_unsafe_tools | false | false | false | false | false | false | false | true | + +### Profile Descriptions + +- **manual** — Human approves every action. Maximum safety, minimum autonomy. +- **review** — Strategy and execution proceed automatically; human reviews before apply. Good default for development. +- **supervised** — Human reviews strategy and execution decisions. Strategy creation itself is automatic. +- **cautious** — Probabilistic gates on most actions. The system proceeds when confident, asks when unsure. +- **trusted** — Automatic for most phases, but human approval required for apply and reversion. Suitable for experienced teams. +- **auto** — Fully automatic except reversion from apply. Suitable for well-tested pipelines. +- **ci** — Designed for CI/CD pipelines. All thresholds at 0.0 but sandbox and checkpoints remain required. +- **full-auto** — No gates, no sandbox, no checkpoints, unsafe tools allowed. Use with extreme caution. + +## Resolution Precedence + +When determining which profile applies to a given plan execution, the system resolves profiles in the following order (highest priority first): + +1. **Plan-level override** — A profile specified directly on the plan. +2. **Project-level setting** — The default profile configured for the project. +3. **Organization-level default** — The organization's default profile. +4. **System default** — Falls back to the `review` built-in profile. + +At each level, the profile may be specified by: +- A built-in name (e.g. `cautious`) +- A namespaced custom profile (e.g. `acme/strict`) + +## Custom Profiles + +Custom profiles use a `namespace/name` naming convention: + +```yaml +name: acme/strict +description: Strict profile for production deployments +schema_version: "1.0" + +auto_strategize: 0.9 +auto_execute: 0.9 +auto_apply: 1.0 +auto_decisions_strategize: 0.8 +auto_decisions_execute: 0.9 +auto_validation_fix: 0.5 +auto_strategy_revision: 0.9 +auto_reversion_from_apply: 1.0 +auto_child_plans: 0.8 +auto_retry_transient: 0.3 +auto_checkpoint_restore: 0.7 + +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false +``` + +See `docs/schema/automation_profile.schema.yaml` for the full YAML schema and `examples/profiles/` for example configurations. diff --git a/docs/schema/automation_profile.schema.yaml b/docs/schema/automation_profile.schema.yaml new file mode 100644 index 000000000..ca0e3d4ef --- /dev/null +++ b/docs/schema/automation_profile.schema.yaml @@ -0,0 +1,110 @@ +# Automation Profile YAML Schema +# Defines the expected structure for automation profile configuration files. +# See docs/specification.md Section "Automation Profiles" for full details. + +type: object +required: + - name +properties: + name: + type: string + minLength: 1 + pattern: "^[a-zA-Z0-9_-]+(/[a-zA-Z0-9_-]+)?$" + description: > + Profile name: bare built-in name (e.g. 'manual') + or namespaced 'namespace/name' (e.g. 'acme/strict') + description: + type: string + description: "Human-readable description of the profile" + schema_version: + type: string + default: "1.0" + description: "Schema version for forward compatibility" + + # Phase-transition thresholds (0.0 = auto, 1.0 = human) + auto_strategize: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic strategy approval" + auto_execute: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic execution approval" + auto_apply: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic apply approval" + + # Decision-autonomy thresholds + auto_decisions_strategize: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic decisions during strategy" + auto_decisions_execute: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic decisions during execution" + + # Self-repair thresholds + auto_validation_fix: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic validation fix" + auto_strategy_revision: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic strategy revision" + auto_reversion_from_apply: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic reversion from apply" + + # Child plan and retry thresholds + auto_child_plans: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic child plan spawning" + auto_retry_transient: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic retry of transient failures" + auto_checkpoint_restore: + type: number + minimum: 0.0 + maximum: 1.0 + default: 0.0 + description: "Threshold for automatic checkpoint restore" + + # Safety requirements + require_sandbox: + type: boolean + default: true + description: "Whether a sandbox is required for execution" + require_checkpoints: + type: boolean + default: true + description: "Whether checkpoints are required" + allow_unsafe_tools: + type: boolean + default: false + description: "Whether unsafe tools may be used" diff --git a/examples/profiles/auto.yaml b/examples/profiles/auto.yaml new file mode 100644 index 000000000..6e23c5206 --- /dev/null +++ b/examples/profiles/auto.yaml @@ -0,0 +1,31 @@ +# Built-in profile: auto +# Fully automatic except reversion. +# Everything proceeds without human gates except reversion from apply. + +name: auto +description: Fully automatic except reversion +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 0.0 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.0 +auto_decisions_execute: 0.0 + +# Self-repair thresholds +auto_validation_fix: 0.0 +auto_strategy_revision: 0.0 +auto_reversion_from_apply: 1.0 + +# Child plan and retry thresholds +auto_child_plans: 0.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 0.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/cautious.yaml b/examples/profiles/cautious.yaml new file mode 100644 index 000000000..eb03cc5dc --- /dev/null +++ b/examples/profiles/cautious.yaml @@ -0,0 +1,32 @@ +# Built-in profile: cautious +# Probabilistic gates on most actions. +# Intermediate thresholds allow the system to proceed automatically +# only when confidence is high. + +name: cautious +description: Probabilistic gates on most actions +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.7 +auto_execute: 0.7 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.6 +auto_decisions_execute: 0.8 + +# Self-repair thresholds +auto_validation_fix: 0.7 +auto_strategy_revision: 0.8 +auto_reversion_from_apply: 0.9 + +# Child plan and retry thresholds +auto_child_plans: 0.7 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 0.6 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/ci.yaml b/examples/profiles/ci.yaml new file mode 100644 index 000000000..3398409ba --- /dev/null +++ b/examples/profiles/ci.yaml @@ -0,0 +1,31 @@ +# Built-in profile: ci +# Designed for CI pipelines. +# All thresholds at 0.0 — fully automatic in a sandboxed environment. + +name: ci +description: Designed for CI pipelines +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 0.0 +auto_apply: 0.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.0 +auto_decisions_execute: 0.0 + +# Self-repair thresholds +auto_validation_fix: 0.0 +auto_strategy_revision: 0.0 +auto_reversion_from_apply: 0.0 + +# Child plan and retry thresholds +auto_child_plans: 0.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 0.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/full-auto.yaml b/examples/profiles/full-auto.yaml new file mode 100644 index 000000000..825f9dd2d --- /dev/null +++ b/examples/profiles/full-auto.yaml @@ -0,0 +1,31 @@ +# Built-in profile: full-auto +# No gates, no sandbox, no checkpoints. +# Maximum autonomy — use with extreme caution. + +name: full-auto +description: No gates, no sandbox, no checkpoints +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 0.0 +auto_apply: 0.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.0 +auto_decisions_execute: 0.0 + +# Self-repair thresholds +auto_validation_fix: 0.0 +auto_strategy_revision: 0.0 +auto_reversion_from_apply: 0.0 + +# Child plan and retry thresholds +auto_child_plans: 0.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 0.0 + +# Safety requirements +require_sandbox: false +require_checkpoints: false +allow_unsafe_tools: true diff --git a/examples/profiles/manual.yaml b/examples/profiles/manual.yaml new file mode 100644 index 000000000..4b0bbe888 --- /dev/null +++ b/examples/profiles/manual.yaml @@ -0,0 +1,31 @@ +# Built-in profile: manual +# Human approves every action. +# All thresholds set to 1.0 — nothing proceeds without explicit approval. + +name: manual +description: Human approves every action +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 1.0 +auto_execute: 1.0 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 1.0 +auto_decisions_execute: 1.0 + +# Self-repair thresholds +auto_validation_fix: 1.0 +auto_strategy_revision: 1.0 +auto_reversion_from_apply: 1.0 + +# Child plan and retry thresholds +auto_child_plans: 1.0 +auto_retry_transient: 1.0 +auto_checkpoint_restore: 1.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/review.yaml b/examples/profiles/review.yaml new file mode 100644 index 000000000..91ed005e1 --- /dev/null +++ b/examples/profiles/review.yaml @@ -0,0 +1,31 @@ +# Built-in profile: review +# Human reviews before apply. +# Strategy and execution proceed automatically; apply requires review. + +name: review +description: Human reviews before apply +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 0.0 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 1.0 +auto_decisions_execute: 1.0 + +# Self-repair thresholds +auto_validation_fix: 1.0 +auto_strategy_revision: 1.0 +auto_reversion_from_apply: 1.0 + +# Child plan and retry thresholds +auto_child_plans: 0.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 1.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/supervised.yaml b/examples/profiles/supervised.yaml new file mode 100644 index 000000000..923307b8b --- /dev/null +++ b/examples/profiles/supervised.yaml @@ -0,0 +1,31 @@ +# Built-in profile: supervised +# Human reviews strategy and execution phases. +# Strategy is automatic; execution and apply require human review. + +name: supervised +description: Human reviews strategy and execution +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 1.0 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.0 +auto_decisions_execute: 1.0 + +# Self-repair thresholds +auto_validation_fix: 1.0 +auto_strategy_revision: 1.0 +auto_reversion_from_apply: 1.0 + +# Child plan and retry thresholds +auto_child_plans: 1.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 1.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/examples/profiles/trusted.yaml b/examples/profiles/trusted.yaml new file mode 100644 index 000000000..89df75b8f --- /dev/null +++ b/examples/profiles/trusted.yaml @@ -0,0 +1,31 @@ +# Built-in profile: trusted +# Auto for most actions, human for apply and revert. +# High autonomy with safety nets at critical boundaries. + +name: trusted +description: Auto for most, human for apply and revert +schema_version: "1.0" + +# Phase-transition thresholds +auto_strategize: 0.0 +auto_execute: 0.0 +auto_apply: 1.0 + +# Decision-autonomy thresholds +auto_decisions_strategize: 0.0 +auto_decisions_execute: 0.0 + +# Self-repair thresholds +auto_validation_fix: 0.0 +auto_strategy_revision: 1.0 +auto_reversion_from_apply: 1.0 + +# Child plan and retry thresholds +auto_child_plans: 0.0 +auto_retry_transient: 0.0 +auto_checkpoint_restore: 1.0 + +# Safety requirements +require_sandbox: true +require_checkpoints: true +allow_unsafe_tools: false diff --git a/features/automation_profile.feature b/features/automation_profile.feature new file mode 100644 index 000000000..a63cb59e9 --- /dev/null +++ b/features/automation_profile.feature @@ -0,0 +1,276 @@ +Feature: Automation Profile Domain Model + As a developer + I want automation profile domain models with threshold validation + So that plan execution autonomy can be configured and enforced + + # ---- Profile validation: valid thresholds ---- + + Scenario: Profile accepts threshold of 0.0 + When I create a profile with auto_strategize 0.0 + Then the profile model should be created + And the profile auto_strategize should be 0.0 + + Scenario: Profile accepts threshold of 0.5 + When I create a profile with auto_strategize 0.5 + Then the profile model should be created + And the profile auto_strategize should be 0.5 + + Scenario: Profile accepts threshold of 1.0 + When I create a profile with auto_strategize 1.0 + Then the profile model should be created + And the profile auto_strategize should be 1.0 + + # ---- Profile validation: invalid thresholds ---- + + Scenario: Profile rejects threshold below 0.0 + When I try to create a profile with auto_strategize -0.1 + Then a profile validation error should be raised + And the profile error should mention "greater than or equal" + + Scenario: Profile rejects threshold above 1.0 + When I try to create a profile with auto_strategize 1.1 + Then a profile validation error should be raised + And the profile error should mention "less than or equal" + + Scenario: Profile rejects auto_execute below 0.0 + When I try to create a profile with auto_execute -0.5 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_execute above 1.0 + When I try to create a profile with auto_execute 2.0 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_apply below 0.0 + When I try to create a profile with auto_apply -0.01 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_decisions_strategize above 1.0 + When I try to create a profile with auto_decisions_strategize 1.5 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_decisions_execute below 0.0 + When I try to create a profile with auto_decisions_execute -1.0 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_validation_fix above 1.0 + When I try to create a profile with auto_validation_fix 9.9 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_strategy_revision below 0.0 + When I try to create a profile with auto_strategy_revision -0.001 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_reversion_from_apply above 1.0 + When I try to create a profile with auto_reversion_from_apply 1.01 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_child_plans below 0.0 + When I try to create a profile with auto_child_plans -0.5 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_retry_transient above 1.0 + When I try to create a profile with auto_retry_transient 100.0 + Then a profile validation error should be raised + + Scenario: Profile rejects auto_checkpoint_restore below 0.0 + When I try to create a profile with auto_checkpoint_restore -0.1 + Then a profile validation error should be raised + + # ---- Built-in profiles load correctly ---- + + Scenario: Built-in manual profile loads with expected values + When I load the built-in profile "manual" + Then the profile model should be created + And the profile auto_strategize should be 1.0 + And the profile auto_execute should be 1.0 + And the profile auto_apply should be 1.0 + And the profile require_sandbox should be true + And the profile require_checkpoints should be true + And the profile allow_unsafe_tools should be false + + Scenario: Built-in review profile loads with expected values + When I load the built-in profile "review" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_execute should be 0.0 + And the profile auto_apply should be 1.0 + And the profile auto_child_plans should be 0.0 + + Scenario: Built-in supervised profile loads with expected values + When I load the built-in profile "supervised" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_execute should be 1.0 + And the profile auto_decisions_strategize should be 0.0 + And the profile auto_child_plans should be 1.0 + + Scenario: Built-in cautious profile loads with expected values + When I load the built-in profile "cautious" + Then the profile model should be created + And the profile auto_strategize should be 0.7 + And the profile auto_execute should be 0.7 + And the profile auto_decisions_strategize should be 0.6 + And the profile auto_decisions_execute should be 0.8 + And the profile auto_validation_fix should be 0.7 + And the profile auto_strategy_revision should be 0.8 + And the profile auto_reversion_from_apply should be 0.9 + And the profile auto_child_plans should be 0.7 + And the profile auto_checkpoint_restore should be 0.6 + + Scenario: Built-in trusted profile loads with expected values + When I load the built-in profile "trusted" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_execute should be 0.0 + And the profile auto_apply should be 1.0 + And the profile auto_strategy_revision should be 1.0 + And the profile auto_reversion_from_apply should be 1.0 + + Scenario: Built-in auto profile loads with expected values + When I load the built-in profile "auto" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_apply should be 1.0 + And the profile auto_reversion_from_apply should be 1.0 + And the profile auto_strategy_revision should be 0.0 + + Scenario: Built-in ci profile loads with expected values + When I load the built-in profile "ci" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_apply should be 0.0 + And the profile require_sandbox should be true + And the profile allow_unsafe_tools should be false + + Scenario: Built-in full-auto profile loads with expected values + When I load the built-in profile "full-auto" + Then the profile model should be created + And the profile auto_strategize should be 0.0 + And the profile auto_apply should be 0.0 + And the profile require_sandbox should be false + And the profile require_checkpoints should be false + And the profile allow_unsafe_tools should be true + + # ---- All 8 built-in profiles exist ---- + + Scenario: All 8 built-in profiles are registered + Then there should be 8 built-in profiles + And built-in profile "manual" should exist + And built-in profile "review" should exist + And built-in profile "supervised" should exist + And built-in profile "cautious" should exist + And built-in profile "trusted" should exist + And built-in profile "auto" should exist + And built-in profile "ci" should exist + And built-in profile "full-auto" should exist + + # ---- Custom profile from YAML dict loads correctly ---- + + Scenario: Custom profile from YAML config loads correctly + When I load a profile from config with name "acme/strict" and auto_apply 0.8 + Then the profile model should be created + And the profile name should be "acme/strict" + And the profile auto_apply should be 0.8 + + Scenario: Profile from config missing name raises error + When I try to load a profile from config missing name + Then a profile config error should be raised with "name" + + # ---- Name format validation ---- + + Scenario: Profile name accepts bare name + When I create a profile with name "manual" + Then the profile model should be created + And the profile name should be "manual" + + Scenario: Profile name accepts namespaced name + When I create a profile with name "acme/strict" + Then the profile model should be created + And the profile name should be "acme/strict" + + Scenario: Profile name rejects spaces + When I try to create a profile with invalid name "bad name" + Then a profile validation error should be raised + And the profile error should mention "Profile name" + + Scenario: Profile name rejects empty string + When I try to create a profile with empty name + Then a profile validation error should be raised + + Scenario: Profile name rejects double slash + When I try to create a profile with invalid name "bad//name" + Then a profile validation error should be raised + + Scenario: Profile name rejects trailing slash + When I try to create a profile with invalid name "bad/" + Then a profile validation error should be raised + + # ---- get_builtin_profile helper ---- + + Scenario: get_builtin_profile returns correct profile + When I call get_builtin_profile with "ci" + Then the profile model should be created + And the profile name should be "ci" + + Scenario: get_builtin_profile raises for unknown name + When I try to call get_builtin_profile with "nonexistent" + Then a profile key error should be raised + + # ---- Schema version ---- + + Scenario: Profile has default schema version + When I create a profile with name "test-profile" + Then the profile schema_version should be "1.0" + + Scenario: Profile accepts custom schema version + When I create a profile with schema_version "2.0" + Then the profile schema_version should be "2.0" + + # ---- Safety field defaults ---- + + Scenario: Profile safety fields have correct defaults + When I create a profile with name "test-defaults" + Then the profile require_sandbox should be true + And the profile require_checkpoints should be true + And the profile allow_unsafe_tools should be false + + # ---- Description field ---- + + Scenario: Profile description defaults to empty string + When I create a profile with name "no-desc" + Then the profile description should be empty + + Scenario: Profile description accepts custom value + When I create a profile with description "My custom profile" + Then the profile description should be "My custom profile" + + # ---- validate_assignment enforcement ---- + + Scenario: Assigning invalid threshold raises error + When I create a profile with name "assign-test" + And I try to assign auto_strategize 1.5 on the profile + Then a profile validation error should be raised + + # ---- Built-in profile retry and checkpoint values ---- + + Scenario: Manual profile has auto_retry_transient 1.0 + When I load the built-in profile "manual" + Then the profile auto_retry_transient should be 1.0 + And the profile auto_checkpoint_restore should be 1.0 + + Scenario: Review profile has auto_retry_transient 0.0 + When I load the built-in profile "review" + Then the profile auto_retry_transient should be 0.0 + And the profile auto_checkpoint_restore should be 1.0 + + Scenario: Cautious profile has auto_retry_transient 0.0 + When I load the built-in profile "cautious" + Then the profile auto_retry_transient should be 0.0 + + # ---- Model dump and round-trip ---- + + Scenario: Profile model_dump produces valid dict + When I create a profile and dump it with auto_apply 0.5 + Then the profile model dump should have key "name" + And the profile model dump should have key "auto_apply" + And the profile model dump auto_apply should be 0.5 diff --git a/features/steps/automation_profile_steps.py b/features/steps/automation_profile_steps.py new file mode 100644 index 000000000..f42170821 --- /dev/null +++ b/features/steps/automation_profile_steps.py @@ -0,0 +1,522 @@ +"""Step definitions for Automation Profile domain model tests.""" + +from typing import Any + +from behave import then, when +from behave.runner import Context +from pydantic import ValidationError + +from cleveragents.domain.models.core.automation_profile import ( + BUILTIN_PROFILES, + AutomationProfile, + get_builtin_profile, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_profile(**overrides: Any) -> AutomationProfile: + """Create a profile with sensible defaults.""" + defaults: dict[str, Any] = { + "name": "test/default", + } + defaults.update(overrides) + return AutomationProfile(**defaults) + + +# --------------------------------------------------------------------------- +# Profile creation with specific thresholds +# --------------------------------------------------------------------------- + + +@when("I create a profile with auto_strategize {value:g}") +def step_create_profile_auto_strategize(context: Context, value: float) -> None: + """Create a profile with a specific auto_strategize.""" + context.profile_model = _make_profile(auto_strategize=value) + context.profile_error = None + + +@when("I try to create a profile with auto_strategize {value:g}") +def step_try_create_profile_auto_strategize(context: Context, value: float) -> None: + """Try creating a profile with invalid auto_strategize.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_strategize=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_execute {value:g}") +def step_try_create_profile_auto_execute(context: Context, value: float) -> None: + """Try creating a profile with invalid auto_execute.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_execute=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_apply {value:g}") +def step_try_create_profile_auto_apply(context: Context, value: float) -> None: + """Try creating with invalid auto_apply.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_apply=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_decisions_strategize {value:g}") +def step_try_create_profile_auto_dec_strat(context: Context, value: float) -> None: + """Try creating with invalid auto_decisions_strategize.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_decisions_strategize=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_decisions_execute {value:g}") +def step_try_create_profile_auto_dec_exec(context: Context, value: float) -> None: + """Try creating with invalid auto_decisions_execute.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_decisions_execute=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_validation_fix {value:g}") +def step_try_create_profile_auto_val_fix(context: Context, value: float) -> None: + """Try creating with invalid auto_validation_fix.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_validation_fix=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_strategy_revision {value:g}") +def step_try_create_profile_auto_strat_rev(context: Context, value: float) -> None: + """Try creating with invalid auto_strategy_revision.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_strategy_revision=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_reversion_from_apply {value:g}") +def step_try_create_profile_auto_rev_apply(context: Context, value: float) -> None: + """Try creating with invalid auto_reversion_from_apply.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_reversion_from_apply=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_child_plans {value:g}") +def step_try_create_profile_auto_child(context: Context, value: float) -> None: + """Try creating with invalid auto_child_plans.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_child_plans=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_retry_transient {value:g}") +def step_try_create_profile_auto_retry(context: Context, value: float) -> None: + """Try creating with invalid auto_retry_transient.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_retry_transient=value) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with auto_checkpoint_restore {value:g}") +def step_try_create_profile_auto_ckpt(context: Context, value: float) -> None: + """Try creating with invalid auto_checkpoint_restore.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = _make_profile(auto_checkpoint_restore=value) + except ValidationError as e: + context.profile_error = e + + +# --------------------------------------------------------------------------- +# Profile assertions +# --------------------------------------------------------------------------- + + +@then("the profile model should be created") +def step_check_profile_created(context: Context) -> None: + """Verify the profile was created.""" + assert context.profile_model is not None, "Profile should be created" + + +@then("the profile auto_strategize should be {expected:g}") +def step_check_auto_strategize(context: Context, expected: float) -> None: + """Check auto_strategize value.""" + actual = context.profile_model.auto_strategize + assert actual == expected, f"Expected auto_strategize {expected}, got {actual}" + + +@then("the profile auto_execute should be {expected:g}") +def step_check_auto_execute(context: Context, expected: float) -> None: + """Check auto_execute value.""" + actual = context.profile_model.auto_execute + assert actual == expected, f"Expected auto_execute {expected}, got {actual}" + + +@then("the profile auto_apply should be {expected:g}") +def step_check_auto_apply(context: Context, expected: float) -> None: + """Check auto_apply value.""" + actual = context.profile_model.auto_apply + assert actual == expected, f"Expected auto_apply {expected}, got {actual}" + + +@then("the profile auto_decisions_strategize should be {expected:g}") +def step_check_auto_dec_strat(context: Context, expected: float) -> None: + """Check auto_decisions_strategize value.""" + actual = context.profile_model.auto_decisions_strategize + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_decisions_execute should be {expected:g}") +def step_check_auto_dec_exec(context: Context, expected: float) -> None: + """Check auto_decisions_execute value.""" + actual = context.profile_model.auto_decisions_execute + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_validation_fix should be {expected:g}") +def step_check_auto_val_fix(context: Context, expected: float) -> None: + """Check auto_validation_fix value.""" + actual = context.profile_model.auto_validation_fix + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_strategy_revision should be {expected:g}") +def step_check_auto_strat_rev(context: Context, expected: float) -> None: + """Check auto_strategy_revision value.""" + actual = context.profile_model.auto_strategy_revision + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_reversion_from_apply should be {expected:g}") +def step_check_auto_rev_apply(context: Context, expected: float) -> None: + """Check auto_reversion_from_apply value.""" + actual = context.profile_model.auto_reversion_from_apply + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_child_plans should be {expected:g}") +def step_check_auto_child(context: Context, expected: float) -> None: + """Check auto_child_plans value.""" + actual = context.profile_model.auto_child_plans + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_retry_transient should be {expected:g}") +def step_check_auto_retry(context: Context, expected: float) -> None: + """Check auto_retry_transient value.""" + actual = context.profile_model.auto_retry_transient + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile auto_checkpoint_restore should be {expected:g}") +def step_check_auto_ckpt(context: Context, expected: float) -> None: + """Check auto_checkpoint_restore value.""" + actual = context.profile_model.auto_checkpoint_restore + assert actual == expected, f"Expected {expected}, got {actual}" + + +@then("the profile require_sandbox should be {expected}") +def step_check_require_sandbox(context: Context, expected: str) -> None: + """Check require_sandbox value.""" + exp_bool = expected.lower() == "true" + actual = context.profile_model.require_sandbox + assert actual is exp_bool, f"Expected require_sandbox {exp_bool}, got {actual}" + + +@then("the profile require_checkpoints should be {expected}") +def step_check_require_checkpoints(context: Context, expected: str) -> None: + """Check require_checkpoints value.""" + exp_bool = expected.lower() == "true" + actual = context.profile_model.require_checkpoints + assert actual is exp_bool, f"Expected require_checkpoints {exp_bool}, got {actual}" + + +@then("the profile allow_unsafe_tools should be {expected}") +def step_check_allow_unsafe(context: Context, expected: str) -> None: + """Check allow_unsafe_tools value.""" + exp_bool = expected.lower() == "true" + actual = context.profile_model.allow_unsafe_tools + assert actual is exp_bool, f"Expected allow_unsafe_tools {exp_bool}, got {actual}" + + +# --------------------------------------------------------------------------- +# Validation errors +# --------------------------------------------------------------------------- + + +@then("a profile validation error should be raised") +def step_check_profile_validation_error( + context: Context, +) -> None: + """Verify that a validation error was raised.""" + assert context.profile_error is not None, "Expected a validation error to be raised" + + +@then('the profile error should mention "{text}"') +def step_check_profile_error_message(context: Context, text: str) -> None: + """Check the error message contains expected text.""" + error_str = str(context.profile_error) + assert text in error_str, f"Expected error to mention '{text}', got: {error_str}" + + +# --------------------------------------------------------------------------- +# Built-in profiles +# --------------------------------------------------------------------------- + + +@when('I load the built-in profile "{name}"') +def step_load_builtin_profile(context: Context, name: str) -> None: + """Load a built-in profile by name.""" + context.profile_model = get_builtin_profile(name) + context.profile_error = None + + +@then("there should be {count:d} built-in profiles") +def step_check_builtin_count(context: Context, count: int) -> None: + """Check the number of built-in profiles.""" + actual = len(BUILTIN_PROFILES) + assert actual == count, f"Expected {count} built-in profiles, got {actual}" + + +@then('built-in profile "{name}" should exist') +def step_check_builtin_exists(context: Context, name: str) -> None: + """Check a built-in profile exists.""" + assert name in BUILTIN_PROFILES, f"Expected built-in profile '{name}' to exist" + + +# --------------------------------------------------------------------------- +# Custom profile from config +# --------------------------------------------------------------------------- + + +@when('I load a profile from config with name "{name}" and auto_apply {value:g}') +def step_load_profile_from_config(context: Context, name: str, value: float) -> None: + """Load a profile from config dict.""" + config = { + "name": name, + "description": "Custom profile", + "auto_apply": value, + } + context.profile_model = AutomationProfile.from_config(config) + context.profile_error = None + + +@when("I try to load a profile from config missing name") +def step_try_load_profile_config_missing_name( + context: Context, +) -> None: + """Try loading profile config without name.""" + context.profile_config_error = None + try: + AutomationProfile.from_config({"description": "No name"}) + except ValueError as e: + context.profile_config_error = e + + +@then('a profile config error should be raised with "{field}"') +def step_check_profile_config_error(context: Context, field: str) -> None: + """Check config error mentions field.""" + assert context.profile_config_error is not None, ( + f"Expected ValueError for missing '{field}'" + ) + assert field in str(context.profile_config_error), ( + f"Expected error to mention '{field}', got: {context.profile_config_error}" + ) + + +# --------------------------------------------------------------------------- +# Name validation +# --------------------------------------------------------------------------- + + +@when('I create a profile with name "{name}"') +def step_create_profile_by_name(context: Context, name: str) -> None: + """Create a profile with specific name.""" + context.profile_model = AutomationProfile(name=name) + context.profile_error = None + + +@when('I try to create a profile with invalid name "{name}"') +def step_try_create_profile_invalid_name(context: Context, name: str) -> None: + """Attempt to create a profile with invalid name.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = AutomationProfile(name=name) + except ValidationError as e: + context.profile_error = e + + +@when("I try to create a profile with empty name") +def step_try_create_profile_empty_name( + context: Context, +) -> None: + """Attempt to create a profile with empty name.""" + context.profile_error = None + context.profile_model = None + try: + context.profile_model = AutomationProfile(name="") + except ValidationError as e: + context.profile_error = e + + +@then('the profile name should be "{expected}"') +def step_check_profile_name(context: Context, expected: str) -> None: + """Check profile name.""" + assert context.profile_model.name == expected, ( + f"Expected name '{expected}', got '{context.profile_model.name}'" + ) + + +# --------------------------------------------------------------------------- +# get_builtin_profile helper +# --------------------------------------------------------------------------- + + +@when('I call get_builtin_profile with "{name}"') +def step_call_get_builtin(context: Context, name: str) -> None: + """Call get_builtin_profile.""" + context.profile_model = get_builtin_profile(name) + context.profile_error = None + + +@when('I try to call get_builtin_profile with "{name}"') +def step_try_call_get_builtin(context: Context, name: str) -> None: + """Try calling get_builtin_profile with bad name.""" + context.profile_key_error = None + try: + get_builtin_profile(name) + except KeyError as e: + context.profile_key_error = e + + +@then("a profile key error should be raised") +def step_check_profile_key_error( + context: Context, +) -> None: + """Verify KeyError was raised.""" + assert context.profile_key_error is not None, "Expected a KeyError to be raised" + + +# --------------------------------------------------------------------------- +# Schema version +# --------------------------------------------------------------------------- + + +@then('the profile schema_version should be "{expected}"') +def step_check_schema_version(context: Context, expected: str) -> None: + """Check profile schema_version.""" + actual = context.profile_model.schema_version + assert actual == expected, f"Expected schema_version '{expected}', got '{actual}'" + + +@when('I create a profile with schema_version "{version}"') +def step_create_profile_with_version(context: Context, version: str) -> None: + """Create a profile with custom schema_version.""" + context.profile_model = AutomationProfile( + name="test-profile", schema_version=version + ) + context.profile_error = None + + +# --------------------------------------------------------------------------- +# Description +# --------------------------------------------------------------------------- + + +@then('the profile description should be "{expected}"') +def step_check_profile_description(context: Context, expected: str) -> None: + """Check profile description.""" + actual = context.profile_model.description + assert actual == expected, f"Expected description '{expected}', got '{actual}'" + + +@then("the profile description should be empty") +def step_check_profile_description_empty( + context: Context, +) -> None: + """Check profile description is empty.""" + actual = context.profile_model.description + assert actual == "", f"Expected empty description, got '{actual}'" + + +@when('I create a profile with description "{desc}"') +def step_create_profile_with_desc(context: Context, desc: str) -> None: + """Create a profile with custom description.""" + context.profile_model = AutomationProfile(name="desc-test", description=desc) + context.profile_error = None + + +# --------------------------------------------------------------------------- +# validate_assignment +# --------------------------------------------------------------------------- + + +@when("I try to assign auto_strategize {value:g} on the profile") +def step_try_assign_threshold(context: Context, value: float) -> None: + """Try to assign an invalid threshold on existing profile.""" + context.profile_error = None + try: + context.profile_model.auto_strategize = value + except ValidationError as e: + context.profile_error = e + + +# --------------------------------------------------------------------------- +# model_dump +# --------------------------------------------------------------------------- + + +@when("I create a profile and dump it with auto_apply {value:g}") +def step_create_profile_with_auto_apply(context: Context, value: float) -> None: + """Create a profile and store model dump.""" + context.profile_model = AutomationProfile(name="dump-test", auto_apply=value) + context.profile_dump = context.profile_model.model_dump() + context.profile_error = None + + +@then('the profile model dump should have key "{key}"') +def step_check_dump_key(context: Context, key: str) -> None: + """Check profile model dump has key.""" + assert key in context.profile_dump, ( + f"Expected key '{key}' in model dump, keys: {list(context.profile_dump)}" + ) + + +@then("the profile model dump auto_apply should be {expected:g}") +def step_check_dump_auto_apply(context: Context, expected: float) -> None: + """Check auto_apply in model dump.""" + actual = context.profile_dump["auto_apply"] + assert actual == expected, f"Expected auto_apply {expected}, got {actual}" diff --git a/implementation_plan.md b/implementation_plan.md index d74314aa6..993905880 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -1992,26 +1992,26 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled **PARALLEL SUBTRACK A6.service [Jeff]**: Profile resolution + precedence **PARALLEL SUBTRACK A6.cli [Jeff]**: CLI commands for profiles **SEQUENTIAL MERGE NOTE**: A6.core must land before A6.service/cli; A6.service must land before gating integration in Section 6. -- [ ] **COMMIT (Owner: Jeff | Group: A6.core | Branch: feature/m4-automation-profiles-core | Planned: Day 22 | Expected: Day 24) - Commit message: "feat(domain): add automation profile model and built-ins"** - - [ ] Git [Jeff]: `git checkout master` - - [ ] Git [Jeff]: `git pull origin master` - - [ ] Git [Jeff]: `git checkout -b feature/m4-automation-profiles-core` - - [ ] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit) - - [ ] Code [Jeff]: Add `AutomationProfile` model with threshold fields per spec (phase transitions, decision autonomy, self-repair, child plan spawning, safety requirements) and validate 0.0-1.0 ranges. - - [ ] Code [Jeff]: Add built-in profiles (`manual`, `review`, `supervised`, `cautious`, `trusted`, `auto`, `ci`, `full-auto`) with exact threshold values per spec and stable names. - - [ ] Code [Jeff]: Add YAML schema for automation profiles under `docs/schema/automation_profile.schema.yaml` and loader helper with env interpolation. - - [ ] Code [Jeff]: Add `examples/profiles/` built-in profile YAMLs (one per profile) with schema version guard. - - [ ] Docs [Jeff]: Add `docs/reference/automation_profiles.md` describing built-ins, threshold semantics, and resolution precedence. - - [ ] Tests (Behave) [Jeff]: Add scenarios for profile validation, built-in defaults, and invalid threshold ranges. - - [ ] Tests (Robot) [Jeff]: Add Robot test that loads each built-in profile and prints summary. - - [ ] Tests (ASV) [Jeff]: Add `benchmarks/automation_profile_bench.py` for profile validation. - - [ ] Quality [Jeff]: Run `nox` (all default sessions, including benchmark). - - [ ] Git [Jeff]: `git add .` - - [ ] Git [Jeff]: `git commit -m "feat(domain): add automation profile model and built-ins"` +- [x] **COMMIT (Owner: Jeff | Group: A6.core | Branch: feature/m4-automation-profiles-core | Planned: Day 22 | Expected: Day 24) - Commit message: "feat(domain): add automation profile model and built-ins"** + - [x] Git [Jeff]: `git checkout master` + - [x] Git [Jeff]: `git pull origin master` + - [x] Git [Jeff]: `git checkout -b feature/m4-automation-profiles-core` + - [x] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit) + - [x] Code [Jeff]: Add `AutomationProfile` model with threshold fields per spec (phase transitions, decision autonomy, self-repair, child plan spawning, safety requirements) and validate 0.0-1.0 ranges. + - [x] Code [Jeff]: Add built-in profiles (`manual`, `review`, `supervised`, `cautious`, `trusted`, `auto`, `ci`, `full-auto`) with exact threshold values per spec and stable names. + - [x] Code [Jeff]: Add YAML schema for automation profiles under `docs/schema/automation_profile.schema.yaml` and loader helper with env interpolation. + - [x] Code [Jeff]: Add `examples/profiles/` built-in profile YAMLs (one per profile) with schema version guard. + - [x] Docs [Jeff]: Add `docs/reference/automation_profiles.md` describing built-ins, threshold semantics, and resolution precedence. + - [x] Tests (Behave) [Jeff]: Add scenarios for profile validation, built-in defaults, and invalid threshold ranges. + - [x] Tests (Robot) [Jeff]: Add Robot test that loads each built-in profile and prints summary. + - [x] Tests (ASV) [Jeff]: Add `benchmarks/automation_profile_bench.py` for profile validation. + - [x] Quality [Jeff]: Run `nox` (all default sessions, including benchmark). + - [x] Git [Jeff]: `git add .` + - [x] Git [Jeff]: `git commit -m "feat(domain): add automation profile model and built-ins"` - [ ] Forgejo PR [Jeff]: Open PR from `feature/m4-automation-profiles-core` to `master` with description "Add automation profile domain model, built-ins, schema, and tests.". - [ ] Git [Jeff]: `git checkout master` - [ ] Git [Jeff]: `git branch -d feature/m4-automation-profiles-core` - - [ ] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. + - [x] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. - [ ] **COMMIT (Owner: Jeff | Group: A6.service | Branch: feature/m4-automation-profiles-service | Planned: Day 23 | Expected: Day 25) - Commit message: "feat(service): resolve automation profiles with precedence"** - [ ] Git [Jeff]: `git checkout master` - [ ] Git [Jeff]: `git pull origin master` diff --git a/robot/automation_profile.robot b/robot/automation_profile.robot new file mode 100644 index 000000000..4f9e3203d --- /dev/null +++ b/robot/automation_profile.robot @@ -0,0 +1,63 @@ +*** Settings *** +Documentation Smoke tests for built-in automation profiles +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER_SCRIPT} robot/helper_automation_profile.py + +*** Test Cases *** +Load Manual Profile + [Documentation] Load the manual built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} manual cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-manual-ok + +Load Review Profile + [Documentation] Load the review built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} review cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-review-ok + +Load Supervised Profile + [Documentation] Load the supervised built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} supervised cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-supervised-ok + +Load Cautious Profile + [Documentation] Load the cautious built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} cautious cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-cautious-ok + +Load Trusted Profile + [Documentation] Load the trusted built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} trusted cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-trusted-ok + +Load Auto Profile + [Documentation] Load the auto built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} auto cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-auto-ok + +Load CI Profile + [Documentation] Load the ci built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} ci cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-ci-ok + +Load Full-Auto Profile + [Documentation] Load the full-auto built-in profile + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} full-auto cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} profile-full-auto-ok + +Profile Summary + [Documentation] Print summary of all built-in profiles + ${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} summary cwd=${WORKSPACE} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} summary-ok diff --git a/robot/helper_automation_profile.py b/robot/helper_automation_profile.py new file mode 100644 index 000000000..e530076b5 --- /dev/null +++ b/robot/helper_automation_profile.py @@ -0,0 +1,88 @@ +"""Helper script for Robot Framework automation profile smoke tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +# Ensure src is importable when run from workspace root +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from cleveragents.domain.models.core.automation_profile import ( + BUILTIN_PROFILES, + AutomationProfile, + get_builtin_profile, +) + +_PROFILE_NAMES = [ + "manual", + "review", + "supervised", + "cautious", + "trusted", + "auto", + "ci", + "full-auto", +] + + +def _test_profile(name: str) -> None: + """Load a built-in profile and verify it.""" + profile = get_builtin_profile(name) + assert isinstance(profile, AutomationProfile) + assert profile.name == name + assert 0.0 <= profile.auto_strategize <= 1.0 + assert 0.0 <= profile.auto_execute <= 1.0 + assert 0.0 <= profile.auto_apply <= 1.0 + print( + f"profile-{name}-ok " + f"(sandbox={profile.require_sandbox}, " + f"checkpoints={profile.require_checkpoints}, " + f"unsafe={profile.allow_unsafe_tools})" + ) + + +def _test_summary() -> None: + """Print summary of all built-in profiles.""" + assert len(BUILTIN_PROFILES) == 8 + for name in _PROFILE_NAMES: + profile = get_builtin_profile(name) + thresholds = [ + profile.auto_strategize, + profile.auto_execute, + profile.auto_apply, + profile.auto_decisions_strategize, + profile.auto_decisions_execute, + profile.auto_validation_fix, + profile.auto_strategy_revision, + profile.auto_reversion_from_apply, + profile.auto_child_plans, + profile.auto_retry_transient, + profile.auto_checkpoint_restore, + ] + avg = sum(thresholds) / len(thresholds) + print( + f" {name:>12}: avg_threshold={avg:.2f} " + f"sandbox={profile.require_sandbox} " + f"checkpoints={profile.require_checkpoints} " + f"unsafe={profile.allow_unsafe_tools}" + ) + print("summary-ok") + + +if __name__ == "__main__": + cmd = sys.argv[1] if len(sys.argv) > 1 else "summary" + dispatch: dict[str, Any] = { + "summary": _test_summary, + } + # Add individual profile tests + for pname in _PROFILE_NAMES: + dispatch[pname] = lambda n=pname: _test_profile(n) + + fn = dispatch.get(cmd) + if fn: + fn() + else: + print(f"Unknown command: {cmd}", file=sys.stderr) + sys.exit(1) diff --git a/src/cleveragents/domain/models/core/__init__.py b/src/cleveragents/domain/models/core/__init__.py index 2a3947206..70f95879a 100644 --- a/src/cleveragents/domain/models/core/__init__.py +++ b/src/cleveragents/domain/models/core/__init__.py @@ -1,6 +1,11 @@ # Action model (ActionState lives here now) from cleveragents.domain.models.core.action import ActionState from cleveragents.domain.models.core.actor import Actor +from cleveragents.domain.models.core.automation_profile import ( + BUILTIN_PROFILES, + AutomationProfile, + get_builtin_profile, +) from cleveragents.domain.models.core.change import ( Change, ChangeSet, @@ -121,8 +126,10 @@ from cleveragents.domain.models.core.tool import ( ) __all__ = [ + "BUILTIN_PROFILES", "ActionState", "Actor", + "AutomationProfile", "BindingMode", "Change", "ChangeSet", @@ -202,5 +209,6 @@ __all__ = [ "Validation", "ValidationMode", "can_transition", + "get_builtin_profile", "parse_namespaced_name", ] diff --git a/src/cleveragents/domain/models/core/automation_profile.py b/src/cleveragents/domain/models/core/automation_profile.py new file mode 100644 index 000000000..8960abba1 --- /dev/null +++ b/src/cleveragents/domain/models/core/automation_profile.py @@ -0,0 +1,415 @@ +"""Automation Profile domain model for CleverAgents v3. + +An **AutomationProfile** defines a set of threshold values that control +how much autonomy the system has at each phase of plan execution. Each +threshold is a float in [0.0, 1.0] where: + +- **0.0** means the action proceeds automatically (no human gate). +- **1.0** means human approval is always required before proceeding. +- Values in between represent a confidence threshold: the system may + proceed autonomously only when its confidence exceeds the threshold. + +## Built-in Profiles + +Eight built-in profiles ship with every installation: + +| Profile | Intent | +|--------------|--------------------------------------------| +| ``manual`` | Human approves every action | +| ``review`` | Human reviews before apply | +| ``supervised``| Human reviews strategy + execution | +| ``cautious`` | Probabilistic gates on most actions | +| ``trusted`` | Auto for most, human for apply + revert | +| ``auto`` | Fully automatic except reversion | +| ``ci`` | Designed for CI pipelines | +| ``full-auto``| No gates, no sandbox, no checkpoints | + +## Profile Naming + +Built-in profiles use bare names (e.g. ``manual``). Custom profiles +use a ``namespace/name`` pattern (e.g. ``acme/strict``). + +Based on ``docs/specification.md`` Section "Automation Profiles" +(lines 13424-13612). +""" + +from __future__ import annotations + +import re +from typing import Any, ClassVar + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +# --------------------------------------------------------------------------- +# Regex patterns +# --------------------------------------------------------------------------- + +_BARE_NAME = re.compile(r"^[a-zA-Z0-9_-]+$") +_NAMESPACED_NAME = re.compile(r"^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$") + + +# --------------------------------------------------------------------------- +# AutomationProfile +# --------------------------------------------------------------------------- + + +class AutomationProfile(BaseModel): + """Domain model for an Automation Profile. + + Controls autonomy thresholds for every phase of plan execution. + All threshold fields are floats in [0.0, 1.0] where 0.0 means + fully automatic and 1.0 means human approval is always required. + """ + + # -- Identity ---------------------------------------------------------- + + name: str = Field( + ..., + min_length=1, + description=( + "Profile name: bare built-in name (e.g. 'manual') " + "or namespaced 'namespace/name' (e.g. 'acme/strict')" + ), + ) + description: str = Field( + "", + description="Human-readable description of the profile", + ) + + # -- Schema version ---------------------------------------------------- + + schema_version: str = Field( + default="1.0", + description="Schema version for forward compatibility", + ) + + # -- Phase-transition thresholds (0.0 = auto, 1.0 = human) ------------ + + auto_strategize: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic strategy approval", + ) + auto_execute: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic execution approval", + ) + auto_apply: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic apply approval", + ) + + # -- Decision-autonomy thresholds -------------------------------------- + + auto_decisions_strategize: float = Field( + 0.0, + ge=0.0, + le=1.0, + description=("Threshold for automatic decisions during strategy"), + ) + auto_decisions_execute: float = Field( + 0.0, + ge=0.0, + le=1.0, + description=("Threshold for automatic decisions during execution"), + ) + + # -- Self-repair thresholds -------------------------------------------- + + auto_validation_fix: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic validation fix", + ) + auto_strategy_revision: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic strategy revision", + ) + auto_reversion_from_apply: float = Field( + 0.0, + ge=0.0, + le=1.0, + description=("Threshold for automatic reversion from apply"), + ) + + # -- Child plan and retry thresholds ----------------------------------- + + auto_child_plans: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic child plan spawning", + ) + auto_retry_transient: float = Field( + 0.0, + ge=0.0, + le=1.0, + description=("Threshold for automatic retry of transient failures"), + ) + auto_checkpoint_restore: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Threshold for automatic checkpoint restore", + ) + + # -- Safety requirements ----------------------------------------------- + + require_sandbox: bool = Field( + True, + description="Whether a sandbox is required for execution", + ) + require_checkpoints: bool = Field( + True, + description="Whether checkpoints are required", + ) + allow_unsafe_tools: bool = Field( + False, + description="Whether unsafe tools may be used", + ) + + # -- Name validation --------------------------------------------------- + + @field_validator("name") + @classmethod + def validate_name_format(cls: type[AutomationProfile], v: str) -> str: + """Validate profile name is bare or namespaced.""" + if not (_BARE_NAME.match(v) or _NAMESPACED_NAME.match(v)): + raise ValueError( + "Profile name must be a bare name " + "(e.g. 'manual') or namespaced " + "'namespace/name' (e.g. 'acme/strict'): " + f"got '{v}'" + ) + return v + + # -- Threshold field validation ---------------------------------------- + + _THRESHOLD_FIELDS: ClassVar[list[str]] = [ + "auto_strategize", + "auto_execute", + "auto_apply", + "auto_decisions_strategize", + "auto_decisions_execute", + "auto_validation_fix", + "auto_strategy_revision", + "auto_reversion_from_apply", + "auto_child_plans", + "auto_retry_transient", + "auto_checkpoint_restore", + ] + + @field_validator( + "auto_strategize", + "auto_execute", + "auto_apply", + "auto_decisions_strategize", + "auto_decisions_execute", + "auto_validation_fix", + "auto_strategy_revision", + "auto_reversion_from_apply", + "auto_child_plans", + "auto_retry_transient", + "auto_checkpoint_restore", + ) + @classmethod + def validate_threshold(cls: type[AutomationProfile], v: float) -> float: + """Validate threshold is in [0.0, 1.0].""" + if v < 0.0 or v > 1.0: + raise ValueError(f"Threshold must be between 0.0 and 1.0, got {v}") + return v + + # -- Factories --------------------------------------------------------- + + @classmethod + def from_config(cls, config: dict[str, Any]) -> AutomationProfile: + """Create an AutomationProfile from a YAML config dict.""" + if "name" not in config: + raise ValueError("AutomationProfile config must include 'name'") + return cls.model_validate(config) + + model_config = ConfigDict( + str_strip_whitespace=True, + validate_assignment=True, + ) + + +# --------------------------------------------------------------------------- +# Built-in profiles +# --------------------------------------------------------------------------- + +BUILTIN_PROFILES: dict[str, AutomationProfile] = { + "manual": AutomationProfile( + name="manual", + description="Human approves every action", + schema_version="1.0", + auto_strategize=1.0, + auto_execute=1.0, + auto_apply=1.0, + auto_decisions_strategize=1.0, + auto_decisions_execute=1.0, + auto_validation_fix=1.0, + auto_strategy_revision=1.0, + auto_reversion_from_apply=1.0, + auto_child_plans=1.0, + auto_retry_transient=1.0, + auto_checkpoint_restore=1.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "review": AutomationProfile( + name="review", + description="Human reviews before apply", + schema_version="1.0", + auto_strategize=0.0, + auto_execute=0.0, + auto_apply=1.0, + auto_decisions_strategize=1.0, + auto_decisions_execute=1.0, + auto_validation_fix=1.0, + auto_strategy_revision=1.0, + auto_reversion_from_apply=1.0, + auto_child_plans=0.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=1.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "supervised": AutomationProfile( + name="supervised", + description=("Human reviews strategy and execution"), + schema_version="1.0", + auto_strategize=0.0, + auto_execute=1.0, + auto_apply=1.0, + auto_decisions_strategize=0.0, + auto_decisions_execute=1.0, + auto_validation_fix=1.0, + auto_strategy_revision=1.0, + auto_reversion_from_apply=1.0, + auto_child_plans=1.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=1.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "cautious": AutomationProfile( + name="cautious", + description=("Probabilistic gates on most actions"), + schema_version="1.0", + auto_strategize=0.7, + auto_execute=0.7, + auto_apply=1.0, + auto_decisions_strategize=0.6, + auto_decisions_execute=0.8, + auto_validation_fix=0.7, + auto_strategy_revision=0.8, + auto_reversion_from_apply=0.9, + auto_child_plans=0.7, + auto_retry_transient=0.0, + auto_checkpoint_restore=0.6, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "trusted": AutomationProfile( + name="trusted", + description=("Auto for most, human for apply and revert"), + schema_version="1.0", + auto_strategize=0.0, + auto_execute=0.0, + auto_apply=1.0, + auto_decisions_strategize=0.0, + auto_decisions_execute=0.0, + auto_validation_fix=0.0, + auto_strategy_revision=1.0, + auto_reversion_from_apply=1.0, + auto_child_plans=0.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=1.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "auto": AutomationProfile( + name="auto", + description="Fully automatic except reversion", + schema_version="1.0", + auto_strategize=0.0, + auto_execute=0.0, + auto_apply=1.0, + auto_decisions_strategize=0.0, + auto_decisions_execute=0.0, + auto_validation_fix=0.0, + auto_strategy_revision=0.0, + auto_reversion_from_apply=1.0, + auto_child_plans=0.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=0.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "ci": AutomationProfile( + name="ci", + description="Designed for CI pipelines", + schema_version="1.0", + auto_strategize=0.0, + auto_execute=0.0, + auto_apply=0.0, + auto_decisions_strategize=0.0, + auto_decisions_execute=0.0, + auto_validation_fix=0.0, + auto_strategy_revision=0.0, + auto_reversion_from_apply=0.0, + auto_child_plans=0.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=0.0, + require_sandbox=True, + require_checkpoints=True, + allow_unsafe_tools=False, + ), + "full-auto": AutomationProfile( + name="full-auto", + description=("No gates, no sandbox, no checkpoints"), + schema_version="1.0", + auto_strategize=0.0, + auto_execute=0.0, + auto_apply=0.0, + auto_decisions_strategize=0.0, + auto_decisions_execute=0.0, + auto_validation_fix=0.0, + auto_strategy_revision=0.0, + auto_reversion_from_apply=0.0, + auto_child_plans=0.0, + auto_retry_transient=0.0, + auto_checkpoint_restore=0.0, + require_sandbox=False, + require_checkpoints=False, + allow_unsafe_tools=True, + ), +} + + +def get_builtin_profile(name: str) -> AutomationProfile: + """Return a built-in profile by name. + + Raises ``KeyError`` if the name is not a recognised built-in. + """ + if name not in BUILTIN_PROFILES: + raise KeyError( + f"Unknown built-in profile '{name}'. " + f"Available: {', '.join(sorted(BUILTIN_PROFILES))}" + ) + return BUILTIN_PROFILES[name] diff --git a/vulture_whitelist.py b/vulture_whitelist.py index 4d552c6b0..8b0356624 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -93,3 +93,12 @@ _resource_type_dict # noqa: B018, F821 _resource_dict # noqa: B018, F821 _print_type_panel # noqa: B018, F821 _get_registry_service # noqa: B018, F821 + +# Automation profile domain model — public API +AutomationProfile # noqa: B018, F821 +BUILTIN_PROFILES # noqa: B018, F821 +get_builtin_profile # noqa: B018, F821 +_THRESHOLD_FIELDS # noqa: B018, F821 +validate_threshold # noqa: B018, F821 +validate_name_format # noqa: B018, F821 +from_config # noqa: B018, F821