refactor(autonomy): rename automation profile task flags to spec names
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 17s
CI / helm (pull_request) Successful in 33s
CI / lint (pull_request) Successful in 3m42s
CI / security (pull_request) Successful in 4m8s
CI / quality (pull_request) Successful in 4m9s
CI / typecheck (pull_request) Successful in 4m20s
CI / integration_tests (pull_request) Successful in 7m2s
CI / unit_tests (pull_request) Successful in 7m55s
CI / docker (pull_request) Successful in 1m19s
CI / coverage (pull_request) Successful in 8m46s
CI / e2e_tests (pull_request) Successful in 16m1s
CI / status-check (pull_request) Successful in 1s
CI / build (push) Successful in 17s
CI / helm (push) Successful in 22s
CI / quality (push) Successful in 31s
CI / lint (push) Successful in 3m28s
CI / typecheck (push) Successful in 3m54s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m5s
CI / integration_tests (push) Successful in 6m13s
CI / unit_tests (push) Successful in 6m28s
CI / docker (push) Successful in 1m34s
CI / coverage (push) Successful in 12m5s
CI / e2e_tests (push) Successful in 18m47s
CI / status-check (push) Successful in 1s
CI / benchmark-publish (push) Successful in 28m0s
CI / benchmark-regression (pull_request) Successful in 59m48s

Renamed all 11 task-type confidence threshold fields in AutomationProfile
from phase-transition semantics to spec-defined task-type semantics.
Updated all 8 built-in profiles, CLI formatting, YAML schema, services,
and all Behave/Robot tests referencing the old field names.

Post-review fixes:
- Fixed 24 stale old field names in M6 fixture files
  (automation_profiles.json, autonomy_guardrails.json)
- Added model_validator(mode='before') to detect legacy field names
  and raise actionable ValueError with rename mapping
- Added semantic bridge comments in PlanLifecycleService mapping
  task-type thresholds to phase-transition gates
- Added threshold_field to structured log messages for observability
- Restored categorised CLI automation-profile show output to match
  spec (Phase Transitions / Decision Automation / Self-Repair /
  Execution Controls) instead of flat list
- Added missing access_network field to spec show output examples
  (Rich, Plain, JSON, YAML variants)
- Aligned ADR-017 profile fields table to all 11 fields with
  descriptions matching spec Automatable Tasks table
- Aligned automation_profiles.md threshold descriptions with spec
- Added spec section references in phase_reversion.md, error_recovery.md,
  and plan_execute.md for field naming context
- Extended repository roundtrip test to assert all 11 threshold fields
- Fixed benchmark _make_profile() passing safety fields as top-level
  kwargs instead of via SafetyProfile sub-model (incompatible with
  extra="forbid")
- Aligned CLI JSON/YAML output structure for automation-profile show
  with the specification grouped format (phase_transitions,
  decision_automation, self_repair, execution_controls)
- Moved safety boolean fields into the Execution Controls section
  of Rich output per spec examples
- Reverted auto profile description to "Fully automatic except apply"
  per specification (line 16703, line 28406)
- Improved bridge comments in test steps with semantic context for
  threshold-to-gate mappings

ISSUES CLOSED: #902
This commit was merged in pull request #1147.
This commit is contained in:
CoreRasurae
2026-03-24 02:26:58 +00:00
committed by Luis Mendes
parent 452a2f35b0
commit 007af498b8
62 changed files with 1678 additions and 1312 deletions
+32 -28
View File
@@ -23,6 +23,7 @@ try:
AutomationProfile,
get_builtin_profile,
)
from cleveragents.domain.models.core.safety_profile import SafetyProfile
except ModuleNotFoundError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cleveragents.domain.models.core.automation_guard import (
@@ -33,6 +34,7 @@ except ModuleNotFoundError:
AutomationProfile,
get_builtin_profile,
)
from cleveragents.domain.models.core.safety_profile import SafetyProfile
def _make_profile() -> AutomationProfile:
@@ -40,20 +42,22 @@ def _make_profile() -> AutomationProfile:
return AutomationProfile(
name="bench/test-profile",
description="Benchmark profile",
auto_strategize=0.7,
auto_execute=0.5,
auto_apply=1.0,
auto_decisions_strategize=0.6,
auto_decisions_execute=0.8,
auto_validation_fix=0.3,
auto_strategy_revision=0.9,
auto_reversion_from_apply=0.4,
auto_child_plans=0.7,
auto_retry_transient=0.1,
auto_checkpoint_restore=0.5,
require_sandbox=True,
require_checkpoints=True,
allow_unsafe_tools=False,
decompose_task=0.7,
create_tool=0.5,
select_tool=1.0,
edit_code=0.6,
execute_command=0.8,
create_file=0.3,
delete_content=0.9,
access_network=0.4,
install_dependency=0.7,
modify_config=0.1,
approve_plan=0.5,
safety=SafetyProfile(
require_sandbox=True,
require_checkpoints=True,
allow_unsafe_tools=False,
),
)
@@ -72,17 +76,17 @@ class ProfileValidationSuite:
"""Benchmark profile with all thresholds at 1.0."""
AutomationProfile(
name="bench/max",
auto_strategize=1.0,
auto_execute=1.0,
auto_apply=1.0,
auto_decisions_strategize=1.0,
auto_decisions_execute=1.0,
auto_validation_fix=1.0,
auto_strategy_revision=1.0,
auto_reversion_from_apply=1.0,
auto_child_plans=1.0,
auto_retry_transient=1.0,
auto_checkpoint_restore=1.0,
decompose_task=1.0,
create_tool=1.0,
select_tool=1.0,
edit_code=1.0,
execute_command=1.0,
create_file=1.0,
delete_content=1.0,
access_network=1.0,
install_dependency=1.0,
modify_config=1.0,
approve_plan=1.0,
)
@@ -110,9 +114,9 @@ class ProfileFromConfigSuite:
self.config = {
"name": "bench/from-config",
"description": "Config benchmark",
"auto_strategize": 0.7,
"auto_execute": 0.5,
"auto_apply": 1.0,
"decompose_task": 0.7,
"create_tool": 0.5,
"select_tool": 1.0,
}
def time_profile_from_config(self) -> None:
+11 -11
View File
@@ -65,17 +65,17 @@ _VALID_YAML = """\
name: bench/test-profile
description: Benchmark test profile
schema_version: "1.0"
auto_strategize: 0.5
auto_execute: 0.5
auto_apply: 1.0
auto_decisions_strategize: 0.5
auto_decisions_execute: 0.5
auto_validation_fix: 0.5
auto_strategy_revision: 0.5
auto_reversion_from_apply: 0.5
auto_child_plans: 0.5
auto_retry_transient: 0.0
auto_checkpoint_restore: 0.5
decompose_task: 0.5
create_tool: 0.5
select_tool: 1.0
edit_code: 0.5
execute_command: 0.5
create_file: 0.5
delete_content: 0.5
access_network: 0.5
install_dependency: 0.5
modify_config: 0.0
approve_plan: 0.5
safety:
require_sandbox: true
require_checkpoints: true
+5 -5
View File
@@ -107,7 +107,7 @@ class EscalationDecisionSuite:
risk_assessment=0.15,
invariant_complexity=0.2,
)
self.operation = OperationContext(operation_type="auto_execute")
self.operation = OperationContext(operation_type="create_tool")
self.profile_cautious = BUILTIN_PROFILES["cautious"]
self.profile_fullauto = BUILTIN_PROFILES["full-auto"]
self.profile_manual = BUILTIN_PROFILES["manual"]
@@ -144,11 +144,11 @@ class HistoricalTrackingSuite:
"""Set up controller with pre-populated history."""
self.controller = AutonomyController()
self.outcome_success = HistoricalOutcome(
operation_type="auto_execute",
operation_type="create_tool",
succeeded=True,
)
self.outcome_failure = HistoricalOutcome(
operation_type="auto_execute",
operation_type="create_tool",
succeeded=False,
)
# Pre-populate history
@@ -161,8 +161,8 @@ class HistoricalTrackingSuite:
def time_get_success_rate(self) -> None:
"""Benchmark retrieving historical success rate."""
self.controller.get_historical_success_rate("auto_execute")
self.controller.get_historical_success_rate("create_tool")
def time_get_history_count(self) -> None:
"""Benchmark retrieving history count."""
self.controller.get_history_count("auto_execute")
self.controller.get_history_count("create_tool")