Files
cleveragents-core/features/steps/autonomy_guardrails_steps.py
T
brent.edwards eb46f0ff54
CI / push-validation (push) Successful in 40s
CI / helm (push) Successful in 44s
CI / build (push) Successful in 1m14s
CI / lint (push) Successful in 1m18s
CI / quality (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 1m19s
CI / typecheck (push) Successful in 2m22s
CI / security (push) Successful in 2m23s
CI / benchmark-regression (push) Failing after 41s
CI / integration_tests (push) Successful in 4m46s
CI / unit_tests (push) Failing after 20m13s
CI / benchmark-publish (push) Failing after 28m28s
CI / coverage (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
fix(plan): add tier hydration and improve architecture review output (#10938)
## Summary

This PR fixes issue #10878 where architecture reviews were truncated because the regex pattern for parsing file output would stop at the first ``` encountered in the Markdown report.

## Changes

- Change file delimiters from ``` to >>>>>>>/<<<<<<< to avoid Markdown conflicts
- Add tier hydration before strategize phase in plan_executor.py
- Increase max_tokens to 16384 in llm_actors.py for longer outputs
- Increase context_max_tokens_hot from 16000 to 32000 in settings.py
- Fix get_hot_view → get_hot_fragments in strategy_actor.py and plan_executor.py
- Add opencode to skip directories in context_tier_hydrator.py
- Change sandbox output location to plan-output/ directory in plan.py
- Add get_context_summary stub method to acms_service.py

## Testing

Run architecture review action and verify the output report is complete with all sections.

Reviewed-on: #10938
2026-05-19 12:43:34 +00:00

1049 lines
33 KiB
Python

"""Step definitions for autonomy guardrails and audit trail scenarios."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock
from behave import given, then, when
from behave.runner import Context
from cleveragents.application.services.autonomy_guardrail_service import (
_MAX_CONFIRMATIONS,
_MAX_METADATA_ENTRIES,
AutonomyGuardrailService,
)
from cleveragents.application.services.plan_executor import (
PlanExecutor,
)
from cleveragents.core.exceptions import PlanError
from cleveragents.domain.models.core.autonomy_guardrails import (
ActorLimits,
AutonomyGuardrails,
GuardrailAuditEntry,
GuardrailAuditTrail,
GuardrailEventType,
GuardrailResult,
)
from cleveragents.domain.models.core.plan import (
PlanPhase,
PlanTimestamps,
ProcessingState,
)
# ---- Model creation steps ----
@when(
"I create autonomy guardrails with max_steps {max_steps:d}"
" and tool_budget {budget:g}"
)
def step_create_guardrails(
context: Context,
max_steps: int,
budget: float,
) -> None:
context.guardrails = AutonomyGuardrails(
max_steps=max_steps,
tool_budget=budget,
)
@when("I create autonomy guardrails with defaults")
def step_create_default_guardrails(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when("I try to create guardrails with max_steps {max_steps:d}")
def step_try_create_bad_max_steps(
context: Context,
max_steps: int,
) -> None:
try:
AutonomyGuardrails(max_steps=max_steps)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@when("I try to create guardrails with tool_budget {budget:g}")
def step_try_create_bad_budget(
context: Context,
budget: float,
) -> None:
try:
AutonomyGuardrails(tool_budget=budget)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@then("the guardrails should have max_steps {value}")
def step_check_max_steps(context: Context, value: str) -> None:
if value == "None":
assert context.guardrails.max_steps is None
else:
assert context.guardrails.max_steps == int(value)
@then("the guardrails should have tool_budget {value}")
def step_check_tool_budget(context: Context, value: str) -> None:
if value == "None":
assert context.guardrails.tool_budget is None
else:
assert context.guardrails.tool_budget == float(value)
@then("the guardrails step_count should be {count:d}")
def step_check_step_count(context: Context, count: int) -> None:
assert context.guardrails.step_count == count
@then("the guardrails budget_spent should be {amount:g}")
def step_check_budget_spent(context: Context, amount: float) -> None:
assert context.guardrails.budget_spent == amount
@then("the guardrails should have empty required_confirmations")
def step_check_empty_confirmations(context: Context) -> None:
assert context.guardrails.required_confirmations == []
@then("a guardrails validation error should be raised")
def step_check_validation_error(context: Context) -> None:
assert context.guardrails_error is not None
@then('the guardrails error should mention "{text}"')
def step_check_error_message(context: Context, text: str) -> None:
assert text in str(context.guardrails_error)
# ---- Step limit checks ----
@given("guardrails with max_steps {max_steps:d}")
def step_given_guardrails_max_steps(
context: Context,
max_steps: int,
) -> None:
context.guardrails = AutonomyGuardrails(max_steps=max_steps)
@given("guardrails with no step limit")
def step_given_no_step_limit(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when("I check step limit at step {step:d}")
def step_check_step_limit(context: Context, step: int) -> None:
context.guardrails.step_count = step
allowed, reason = context.guardrails.check_step_limit()
context.step_allowed = allowed
context.step_reason = reason
@then("the step check should be allowed")
def step_check_allowed(context: Context) -> None:
assert context.step_allowed is True
@then("the step check should be denied")
def step_check_denied(context: Context) -> None:
assert context.step_allowed is False
# ---- Budget checks ----
@given("guardrails with tool_budget {budget:g}")
def step_given_guardrails_budget(
context: Context,
budget: float,
) -> None:
context.guardrails = AutonomyGuardrails(tool_budget=budget)
@given("guardrails with tool_budget {budget:g} and budget_spent {spent:g}")
def step_given_guardrails_budget_spent(
context: Context,
budget: float,
spent: float,
) -> None:
context.guardrails = AutonomyGuardrails(
tool_budget=budget,
budget_spent=spent,
)
@given("guardrails with no budget limit")
def step_given_no_budget_limit(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when("I check tool budget for cost {cost:g}")
def step_check_budget(context: Context, cost: float) -> None:
allowed, reason = context.guardrails.check_tool_budget(cost)
context.budget_allowed = allowed
context.budget_reason = reason
# Provide a BudgetCheckResult for shared @then steps (#584)
from cleveragents.domain.models.core.cost_budget import BudgetCheckResult
context.budget_result = BudgetCheckResult(allowed=allowed, reason=reason or "")
@when("I try to check tool budget for cost {cost:g}")
def step_try_check_budget_negative(
context: Context,
cost: float,
) -> None:
try:
context.guardrails.check_tool_budget(cost)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
# "the budget check should be allowed/denied" steps moved to
# cost_budgets_steps.py (shared across both feature files, #584).
# ---- Confirmation checks ----
@given('guardrails with required confirmations "{confirmations}"')
def step_given_confirmations(
context: Context,
confirmations: str,
) -> None:
context.guardrails = AutonomyGuardrails(
required_confirmations=confirmations.split(","),
)
@given("guardrails with no required confirmations")
def step_given_no_confirmations(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when('I check confirmation for operation "{operation}"')
def step_check_confirmation(context: Context, operation: str) -> None:
required, reason = context.guardrails.check_confirmation_required(operation)
context.confirmation_required = required
context.confirmation_reason = reason
@then("confirmation should be required")
def step_confirmation_required(context: Context) -> None:
assert context.confirmation_required is True
@then("confirmation should not be required")
def step_confirmation_not_required(context: Context) -> None:
assert context.confirmation_required is False
# ---- Step counter and budget tracking ----
@when("I increment the step counter {count:d} times")
def step_increment_counter(context: Context, count: int) -> None:
for _ in range(count):
context.guardrails.increment_step()
@then("the step_count should be {count:d}")
def step_verify_step_count(context: Context, count: int) -> None:
assert context.guardrails.step_count == count
@when("I record a tool cost of {cost:g}")
def step_record_cost(context: Context, cost: float) -> None:
context.guardrails.record_cost(cost)
@then("the budget_spent should be {amount:g}")
def step_verify_budget_spent(context: Context, amount: float) -> None:
assert abs(context.guardrails.budget_spent - amount) < 0.001
@when("I try to record a negative cost")
def step_try_record_negative_cost(context: Context) -> None:
try:
context.guardrails.record_cost(-5.0)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
# ---- Audit trail ----
@given("an empty audit trail")
def step_given_empty_trail(context: Context) -> None:
context.audit_trail = GuardrailAuditTrail()
@when("I add a step_allowed entry to the audit trail")
def step_add_allowed_entry(context: Context) -> None:
entry = GuardrailAuditEntry(
event_type=GuardrailEventType.STEP_ALLOWED,
guard_name="step_limit",
result=GuardrailResult.ALLOWED,
)
context.audit_trail.add_entry(entry)
@when("I add a budget_blocked entry to the audit trail")
def step_add_blocked_entry(context: Context) -> None:
entry = GuardrailAuditEntry(
event_type=GuardrailEventType.BUDGET_BLOCKED,
guard_name="tool_budget",
result=GuardrailResult.DENIED,
reason="Budget exceeded",
)
context.audit_trail.add_entry(entry)
@then("the audit trail should have {count:d} entries")
def step_check_trail_count(context: Context, count: int) -> None:
trail = getattr(context, "audit_trail", None)
if trail is None:
trail = context.service_trail
assert len(trail.entries) == count
@then("the audit trail allowed_count should be {count:d}")
def step_check_allowed_count(context: Context, count: int) -> None:
assert context.audit_trail.allowed_count == count
@then("the audit trail denied_count should be {count:d}")
def step_check_denied_count(context: Context, count: int) -> None:
assert context.audit_trail.denied_count == count
@when(
"I create an audit entry with event_type {event_type} and guard_name {guard_name}"
)
def step_create_entry(
context: Context,
event_type: str,
guard_name: str,
) -> None:
context.audit_entry = GuardrailAuditEntry(
event_type=GuardrailEventType(event_type),
guard_name=guard_name,
result=GuardrailResult.DENIED,
reason="Test reason",
)
@then('the audit entry event_type should be "{value}"')
def step_check_entry_event_type(context: Context, value: str) -> None:
assert context.audit_entry.event_type.value == value
@then('the audit entry guard_name should be "{value}"')
def step_check_entry_guard_name(context: Context, value: str) -> None:
assert context.audit_entry.guard_name == value
@then('the audit entry result should be "{value}"')
def step_check_entry_result(context: Context, value: str) -> None:
assert context.audit_entry.result.value == value
@then("the audit entry should have a timestamp")
def step_check_entry_timestamp(context: Context) -> None:
assert context.audit_entry.timestamp is not None
assert len(context.audit_entry.timestamp) > 0
# ---- Service steps ----
@given("an autonomy guardrail service")
def step_given_service(context: Context) -> None:
context.guardrail_service = AutonomyGuardrailService()
@when('I configure guardrails for plan "{plan_id}" with max_steps {max_steps:d}')
def step_configure_guardrails(
context: Context,
plan_id: str,
max_steps: int,
) -> None:
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(max_steps=max_steps),
)
@then('the service should return guardrails for plan "{plan_id}"')
def step_service_has_guardrails(
context: Context,
plan_id: str,
) -> None:
g = context.guardrail_service.get_guardrails(plan_id)
assert g is not None
context.service_guardrails = g
@then('the service should return None for plan "{plan_id}"')
def step_service_no_guardrails(
context: Context,
plan_id: str,
) -> None:
g = context.guardrail_service.get_guardrails(plan_id)
assert g is None
@then("the service guardrails should have max_steps {value:d}")
def step_service_max_steps(context: Context, value: int) -> None:
assert context.service_guardrails.max_steps == value
@given('an autonomy guardrail service with guardrails for plan "{plan_id}"')
def step_given_service_with_guardrails(
context: Context,
plan_id: str,
) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(max_steps=10),
)
@given('an autonomy guardrail service with budget guardrails for plan "{plan_id}"')
def step_given_service_with_budget(
context: Context,
plan_id: str,
) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(tool_budget=100.0),
)
@given(
'an autonomy guardrail service with confirmation guardrails for plan "{plan_id}"'
)
def step_given_service_with_confirmations(
context: Context,
plan_id: str,
) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(required_confirmations=["deploy", "delete"]),
)
@when('I check step limit via service for plan "{plan_id}" at step {step:d}')
def step_service_check_step(
context: Context,
plan_id: str,
step: int,
) -> None:
context.step_result = context.guardrail_service.check_step_limit(plan_id, step)
@when('I check tool budget via service for plan "{plan_id}" with cost {cost:g}')
def step_service_check_budget(
context: Context,
plan_id: str,
cost: float,
) -> None:
context.budget_result = context.guardrail_service.check_tool_budget(plan_id, cost)
@when('I check confirmation via service for plan "{plan_id}" operation "{operation}"')
def step_service_check_confirmation(
context: Context,
plan_id: str,
operation: str,
) -> None:
context.confirmation_result = context.guardrail_service.check_confirmation_required(
plan_id, operation
)
@then('the audit trail for plan "{plan_id}" should have {count:d} entry')
def step_service_trail_count(
context: Context,
plan_id: str,
count: int,
) -> None:
trail = context.guardrail_service.get_audit_trail(plan_id)
context.service_trail = trail
assert len(trail.entries) == count
@then("the service guardrails budget_spent should be {amount:g}")
def step_service_budget_spent(context: Context, amount: float) -> None:
# Use the plan_id stored by the service budget step (plan-003)
guardrails = context.guardrail_service.get_guardrails("plan-003")
assert guardrails is not None, "No budget guardrails found for plan-003"
assert abs(guardrails.budget_spent - amount) < 0.001
@then('the metadata for plan "{plan_id}" should contain the audit trail')
def step_service_metadata(context: Context, plan_id: str) -> None:
metadata = context.guardrail_service.get_audit_trail_as_metadata(plan_id)
assert "guardrail_audit_trail" in metadata
assert "autonomy_guardrails" in metadata
@when('I load guardrails from metadata for plan "{plan_id}"')
def step_load_metadata(context: Context, plan_id: str) -> None:
metadata = {
"autonomy_guardrails": {
"max_steps": 20,
"tool_budget": 50.0,
"required_confirmations": [],
"step_count": 0,
"budget_spent": 0.0,
},
"guardrail_audit_trail": {"entries": []},
}
context.guardrail_service.load_from_metadata(plan_id, metadata)
@when('I remove plan "{plan_id}" from the service')
def step_remove_plan(context: Context, plan_id: str) -> None:
context.guardrail_service.remove_plan(plan_id)
@then("the step limit check should return True")
def step_check_returns_true(context: Context) -> None:
assert context.step_result is True
@then("the tool budget check should return True")
def step_budget_returns_true(context: Context) -> None:
assert context.budget_result is True
@then("the confirmation check should return False")
def step_confirmation_returns_false(context: Context) -> None:
assert context.confirmation_result is False
@when("I try to configure guardrails with empty plan_id")
def step_try_empty_plan_id(context: Context) -> None:
try:
context.guardrail_service.configure_guardrails(
"",
AutonomyGuardrails(),
)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@then("GuardrailEventType should have at least {count:d} members")
def step_check_event_type_members(
context: Context,
count: int,
) -> None:
assert len(GuardrailEventType) >= count
@then("GuardrailResult should have exactly {count:d} members")
def step_check_result_members(context: Context, count: int) -> None:
assert len(GuardrailResult) == count
# ---- Wall-clock time steps ----
@given("guardrails with no wall-clock limit")
def step_given_no_wall_clock(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@given("guardrails with wall-clock limit {seconds:g} seconds started recently")
def step_given_wall_clock_recent(context: Context, seconds: float) -> None:
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=seconds)
context.guardrails.mark_started()
@given("guardrails with wall-clock limit that has already expired")
def step_given_wall_clock_expired(context: Context) -> None:
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=0.001)
# Set start_time far in the past to guarantee expiry
context.guardrails.start_time = (
datetime.now(tz=UTC) - timedelta(seconds=10)
).isoformat()
@when("I check wall-clock time")
def step_check_wall_clock(context: Context) -> None:
allowed, reason = context.guardrails.check_wall_clock()
context.wall_clock_allowed = allowed
context.wall_clock_reason = reason
@then("the wall-clock check should be allowed")
def step_wall_clock_allowed(context: Context) -> None:
assert context.wall_clock_allowed is True
@then("the wall-clock check should be denied")
def step_wall_clock_denied(context: Context) -> None:
assert context.wall_clock_allowed is False
@given('an autonomy guardrail service with wall-clock guardrails for plan "{plan_id}"')
def step_given_service_wall_clock(context: Context, plan_id: str) -> None:
context.guardrail_service = AutonomyGuardrailService()
g = AutonomyGuardrails(max_wall_clock_seconds=600.0)
g.mark_started()
context.guardrail_service.configure_guardrails(plan_id, g)
@when('I check wall-clock via service for plan "{plan_id}"')
def step_service_check_wall_clock(context: Context, plan_id: str) -> None:
context.wall_clock_result = context.guardrail_service.check_wall_clock(plan_id)
# ---- Actor tool-call limit steps ----
@given("guardrails with actor tool-call limit {limit:d}")
def step_given_actor_limit(context: Context, limit: int) -> None:
context.guardrails = AutonomyGuardrails(
actor_limits=ActorLimits(max_tool_calls_per_invocation=limit),
)
@given("guardrails with no actor tool-call limit")
def step_given_no_actor_limit(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when("I check actor tool calls with {calls:d} current calls")
def step_check_actor_calls(context: Context, calls: int) -> None:
allowed, reason = context.guardrails.check_actor_tool_calls(calls)
context.actor_allowed = allowed
context.actor_reason = reason
@then("the actor tool-call check should be allowed")
def step_actor_allowed(context: Context) -> None:
assert context.actor_allowed is True
@then("the actor tool-call check should be denied")
def step_actor_denied(context: Context) -> None:
assert context.actor_allowed is False
@given('an autonomy guardrail service with actor-limit guardrails for plan "{plan_id}"')
def step_given_service_actor_limit(context: Context, plan_id: str) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(
actor_limits=ActorLimits(max_tool_calls_per_invocation=10),
),
)
@when('I check actor tool calls via service for plan "{plan_id}" with {calls:d} calls')
def step_service_check_actor_calls(
context: Context,
plan_id: str,
calls: int,
) -> None:
context.actor_result = context.guardrail_service.check_actor_tool_calls(
plan_id, calls
)
# ---- Bounded audit trail eviction steps ----
@given("an audit trail with max_entries {max_entries:d}")
def step_given_bounded_trail(context: Context, max_entries: int) -> None:
context.audit_trail = GuardrailAuditTrail(max_entries=max_entries)
@when("I add {count:d} step_allowed entries to the audit trail")
def step_add_many_allowed_entries(context: Context, count: int) -> None:
for _ in range(count):
entry = GuardrailAuditEntry(
event_type=GuardrailEventType.STEP_ALLOWED,
guard_name="step_limit",
result=GuardrailResult.ALLOWED,
)
context.audit_trail.add_entry(entry)
# ---- Edge-case steps ----
@when('I try to load metadata with oversized audit trail for plan "{plan_id}"')
def step_try_load_oversized_trail(context: Context, plan_id: str) -> None:
metadata = {
"guardrail_audit_trail": {
"entries": [{}] * (_MAX_METADATA_ENTRIES + 1),
},
}
try:
context.guardrail_service.load_from_metadata(plan_id, metadata)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@when('I try to load metadata with oversized confirmations for plan "{plan_id}"')
def step_try_load_oversized_confirmations(
context: Context,
plan_id: str,
) -> None:
metadata = {
"autonomy_guardrails": {
"max_steps": 10,
"tool_budget": 50.0,
"required_confirmations": ["op"] * (_MAX_CONFIRMATIONS + 1),
"step_count": 0,
"budget_spent": 0.0,
},
}
try:
context.guardrail_service.load_from_metadata(plan_id, metadata)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@when('I serialize and restore metadata for plan "{plan_id}"')
def step_round_trip_metadata(context: Context, plan_id: str) -> None:
metadata = context.guardrail_service.get_audit_trail_as_metadata(plan_id)
context.round_trip_metadata = metadata
# Create a fresh service and restore
context.restored_service = AutonomyGuardrailService()
context.restored_service.load_from_metadata(plan_id, metadata)
@then('the restored guardrails for plan "{plan_id}" should match the original')
def step_verify_round_trip(context: Context, plan_id: str) -> None:
original = context.guardrail_service.get_guardrails(plan_id)
restored = context.restored_service.get_guardrails(plan_id)
assert original is not None
assert restored is not None
assert restored.max_steps == original.max_steps
assert restored.tool_budget == original.tool_budget
assert restored.required_confirmations == original.required_confirmations
assert restored.step_count == original.step_count
# Check audit trail was also restored
original_trail = context.guardrail_service.get_audit_trail(plan_id)
restored_trail = context.restored_service.get_audit_trail(plan_id)
assert len(restored_trail.entries) == len(original_trail.entries)
# ---- start_time validation steps ----
@when("I create guardrails with a valid start_time")
def step_create_guardrails_valid_start_time(context: Context) -> None:
context.guardrails = AutonomyGuardrails(
start_time=datetime.now(tz=UTC).isoformat(),
)
@when('I try to create guardrails with start_time "{value}"')
def step_try_create_bad_start_time(context: Context, value: str) -> None:
try:
AutonomyGuardrails(start_time=value)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@then("the guardrails start_time should be set")
def step_check_start_time_set(context: Context) -> None:
assert context.guardrails.start_time is not None
assert len(context.guardrails.start_time) > 0
@then("the guardrails start_time should be None")
def step_check_start_time_none(context: Context) -> None:
assert context.guardrails.start_time is None
@given("guardrails with a malformed start_time bypassing validation")
def step_given_malformed_start_time(context: Context) -> None:
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=60.0)
# Bypass Pydantic validation by setting directly via object.__setattr__
object.__setattr__(context.guardrails, "start_time", "not-a-date")
# ---- Retry-per-failure check steps ----
@given("guardrails with retry limit {limit:d}")
def step_given_retry_limit(context: Context, limit: int) -> None:
context.guardrails = AutonomyGuardrails(
actor_limits=ActorLimits(max_retries_per_failure=limit),
)
@given("guardrails with no retry limit")
def step_given_no_retry_limit(context: Context) -> None:
context.guardrails = AutonomyGuardrails()
@when("I check retries per failure with {retries:d} current retries")
def step_check_retries(context: Context, retries: int) -> None:
allowed, reason = context.guardrails.check_retries_per_failure(retries)
context.retry_allowed = allowed
context.retry_reason = reason
@then("the retry check should be allowed")
def step_retry_allowed(context: Context) -> None:
assert context.retry_allowed is True
@then("the retry check should be denied")
def step_retry_denied(context: Context) -> None:
assert context.retry_allowed is False
@given('an autonomy guardrail service with retry-limit guardrails for plan "{plan_id}"')
def step_given_service_retry_limit(context: Context, plan_id: str) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(
actor_limits=ActorLimits(max_retries_per_failure=3),
),
)
@when('I check retries via service for plan "{plan_id}" with {retries:d} retries')
def step_service_check_retries(
context: Context,
plan_id: str,
retries: int,
) -> None:
context.retry_result = context.guardrail_service.check_retries_per_failure(
plan_id, retries
)
@then("the retry service check should return False")
def step_retry_service_returns_false(context: Context) -> None:
assert context.retry_result is False
@given(
'a guardrail service with max_retries_per_failure {limit:d} for plan "{plan_id}"'
)
def step_given_service_custom_retry_limit(
context: Context, limit: int, plan_id: str
) -> None:
context.guardrail_service = AutonomyGuardrailService()
context.guardrail_service.configure_guardrails(
plan_id,
AutonomyGuardrails(
actor_limits=ActorLimits(max_retries_per_failure=limit),
),
)
@when('I check retries for plan "{plan_id}" incrementing from {start:d} to {end:d}')
def step_check_retries_incrementing(
context: Context, plan_id: str, start: int, end: int
) -> None:
context.retry_results = []
for i in range(start, end + 1):
allowed = context.guardrail_service.check_retries_per_failure(plan_id, i)
context.retry_results.append((i, allowed))
@then("retry checks {allowed_csv} should be allowed")
def step_retry_checks_allowed(context: Context, allowed_csv: str) -> None:
expected = [int(x.strip()) for x in allowed_csv.split(",")]
results_map = dict(context.retry_results)
for idx in expected:
assert results_map[idx] is True, (
f"Expected retry {idx} to be allowed, but it was denied"
)
@then("retry checks {denied_csv} should be denied")
def step_retry_checks_denied(context: Context, denied_csv: str) -> None:
expected = [int(x.strip()) for x in denied_csv.split(",")]
results_map = dict(context.retry_results)
for idx in expected:
assert results_map[idx] is False, (
f"Expected retry {idx} to be denied, but it was allowed"
)
# ---- PlanExecutor guardrail integration helpers ----
def _make_guardrail_mock_plan(
phase: PlanPhase = PlanPhase.EXECUTE,
state: ProcessingState = ProcessingState.QUEUED,
definition_of_done: str = "Step A\nStep B",
decision_root_id: str = "root-001",
) -> MagicMock:
"""Create a mock plan suitable for PlanExecutor tests."""
plan = MagicMock()
plan.phase = phase
plan.state = state
plan.definition_of_done = definition_of_done
plan.decision_root_id = decision_root_id
plan.timestamps = PlanTimestamps()
plan.error_details = {}
plan.changeset_id = None
plan.sandbox_refs = []
plan.read_only = False
plan.invariants = []
return plan
def _make_guardrail_mock_lifecycle(plan: MagicMock) -> MagicMock:
"""Create a mock lifecycle service for PlanExecutor tests."""
lcs = MagicMock()
lcs.get_plan.return_value = plan
lcs.start_strategize = MagicMock()
lcs.complete_strategize = MagicMock()
lcs.fail_strategize = MagicMock()
lcs.start_execute = MagicMock()
lcs.complete_execute = MagicMock()
lcs.fail_execute = MagicMock()
lcs.commit_plan = MagicMock()
return lcs
# ---- PlanExecutor guardrail integration steps ----
@given("a plan executor with guardrail service and step limit {limit:d}")
def step_given_executor_with_step_limit(context: Context, limit: int) -> None:
service = AutonomyGuardrailService()
service.configure_guardrails(
"grd-plan-001",
AutonomyGuardrails(max_steps=limit),
)
context.guardrail_plan_id = "grd-plan-001"
context.guardrail_exec_service = service
@given("a plan executor with guardrail service and expired wall-clock")
def step_given_executor_with_expired_wall_clock(context: Context) -> None:
service = AutonomyGuardrailService()
g = AutonomyGuardrails(max_wall_clock_seconds=0.001)
g.start_time = (datetime.now(tz=UTC) - timedelta(seconds=10)).isoformat()
service.configure_guardrails("grd-plan-002", g)
context.guardrail_plan_id = "grd-plan-002"
context.guardrail_exec_service = service
@given("a plan executor without guardrail service")
def step_given_executor_no_guardrails(context: Context) -> None:
context.guardrail_plan_id = "grd-plan-003"
context.guardrail_exec_service = None
@given("a plan executor with guardrail service and no wall-clock limit")
def step_given_executor_no_wall_clock(context: Context) -> None:
service = AutonomyGuardrailService()
service.configure_guardrails(
"grd-plan-004",
AutonomyGuardrails(max_steps=100),
)
context.guardrail_plan_id = "grd-plan-004"
context.guardrail_exec_service = service
@given("a plan in execute phase with {count:d} decisions")
def step_given_plan_in_execute_phase(context: Context, count: int) -> None:
steps = "\n".join(f"Step {chr(65 + i)}" for i in range(count))
plan = _make_guardrail_mock_plan(definition_of_done=steps)
lifecycle = _make_guardrail_mock_lifecycle(plan)
context.guardrail_executor = PlanExecutor(
lifecycle_service=lifecycle,
guardrail_service=context.guardrail_exec_service,
)
@when("I run execute on the plan")
def step_run_execute(context: Context) -> None:
try:
context.execute_result = context.guardrail_executor.run_execute(
context.guardrail_plan_id,
)
context.execute_error = None
except Exception as exc:
context.execute_error = exc
context.execute_result = None
@then("execution should fail with a guardrail step limit error")
def step_check_step_limit_error(context: Context) -> None:
assert context.execute_error is not None
assert isinstance(context.execute_error, PlanError)
assert "step limit" in str(context.execute_error).lower()
@then("execution should fail with a guardrail wall-clock error")
def step_check_wall_clock_error(context: Context) -> None:
assert context.execute_error is not None
assert isinstance(context.execute_error, PlanError)
assert "wall-clock" in str(context.execute_error).lower()
@then("execution should succeed")
def step_check_execution_success(context: Context) -> None:
assert context.execute_error is None
assert context.execute_result is not None
@then("the guardrails start_time should have been set")
def step_check_start_time_was_set(context: Context) -> None:
service = context.guardrail_exec_service
assert service is not None
guardrails = service.get_guardrails(context.guardrail_plan_id)
assert guardrails is not None
assert guardrails.start_time is not None
# ---- Additional validation edge-case steps ----
@when("I try to create actor limits with max_tool_calls_per_invocation {value:d}")
def step_try_bad_max_tool_calls(context: Context, value: int) -> None:
try:
ActorLimits(max_tool_calls_per_invocation=value)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@when("I try to create actor limits with max_retries_per_failure {value:d}")
def step_try_bad_max_retries(context: Context, value: int) -> None:
try:
ActorLimits(max_retries_per_failure=value)
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc
@when("I try to create guardrails with max_wall_clock_seconds {value:d}")
def step_try_bad_wall_clock(context: Context, value: int) -> None:
try:
AutonomyGuardrails(max_wall_clock_seconds=float(value))
context.guardrails_error = None
except Exception as exc:
context.guardrails_error = exc