eb46f0ff54
CI / push-validation (push) Successful in 40s
CI / helm (push) Successful in 44s
CI / build (push) Successful in 1m14s
CI / lint (push) Successful in 1m18s
CI / quality (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 1m19s
CI / typecheck (push) Successful in 2m22s
CI / security (push) Successful in 2m23s
CI / benchmark-regression (push) Failing after 41s
CI / integration_tests (push) Successful in 4m46s
CI / unit_tests (push) Failing after 20m13s
CI / benchmark-publish (push) Failing after 28m28s
CI / coverage (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
## Summary This PR fixes issue #10878 where architecture reviews were truncated because the regex pattern for parsing file output would stop at the first ``` encountered in the Markdown report. ## Changes - Change file delimiters from ``` to >>>>>>>/<<<<<<< to avoid Markdown conflicts - Add tier hydration before strategize phase in plan_executor.py - Increase max_tokens to 16384 in llm_actors.py for longer outputs - Increase context_max_tokens_hot from 16000 to 32000 in settings.py - Fix get_hot_view → get_hot_fragments in strategy_actor.py and plan_executor.py - Add opencode to skip directories in context_tier_hydrator.py - Change sandbox output location to plan-output/ directory in plan.py - Add get_context_summary stub method to acms_service.py ## Testing Run architecture review action and verify the output report is complete with all sections. Reviewed-on: #10938
1049 lines
33 KiB
Python
1049 lines
33 KiB
Python
"""Step definitions for autonomy guardrails and audit trail scenarios."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.application.services.autonomy_guardrail_service import (
|
|
_MAX_CONFIRMATIONS,
|
|
_MAX_METADATA_ENTRIES,
|
|
AutonomyGuardrailService,
|
|
)
|
|
from cleveragents.application.services.plan_executor import (
|
|
PlanExecutor,
|
|
)
|
|
from cleveragents.core.exceptions import PlanError
|
|
from cleveragents.domain.models.core.autonomy_guardrails import (
|
|
ActorLimits,
|
|
AutonomyGuardrails,
|
|
GuardrailAuditEntry,
|
|
GuardrailAuditTrail,
|
|
GuardrailEventType,
|
|
GuardrailResult,
|
|
)
|
|
from cleveragents.domain.models.core.plan import (
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
)
|
|
|
|
# ---- Model creation steps ----
|
|
|
|
|
|
@when(
|
|
"I create autonomy guardrails with max_steps {max_steps:d}"
|
|
" and tool_budget {budget:g}"
|
|
)
|
|
def step_create_guardrails(
|
|
context: Context,
|
|
max_steps: int,
|
|
budget: float,
|
|
) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
max_steps=max_steps,
|
|
tool_budget=budget,
|
|
)
|
|
|
|
|
|
@when("I create autonomy guardrails with defaults")
|
|
def step_create_default_guardrails(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when("I try to create guardrails with max_steps {max_steps:d}")
|
|
def step_try_create_bad_max_steps(
|
|
context: Context,
|
|
max_steps: int,
|
|
) -> None:
|
|
try:
|
|
AutonomyGuardrails(max_steps=max_steps)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@when("I try to create guardrails with tool_budget {budget:g}")
|
|
def step_try_create_bad_budget(
|
|
context: Context,
|
|
budget: float,
|
|
) -> None:
|
|
try:
|
|
AutonomyGuardrails(tool_budget=budget)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@then("the guardrails should have max_steps {value}")
|
|
def step_check_max_steps(context: Context, value: str) -> None:
|
|
if value == "None":
|
|
assert context.guardrails.max_steps is None
|
|
else:
|
|
assert context.guardrails.max_steps == int(value)
|
|
|
|
|
|
@then("the guardrails should have tool_budget {value}")
|
|
def step_check_tool_budget(context: Context, value: str) -> None:
|
|
if value == "None":
|
|
assert context.guardrails.tool_budget is None
|
|
else:
|
|
assert context.guardrails.tool_budget == float(value)
|
|
|
|
|
|
@then("the guardrails step_count should be {count:d}")
|
|
def step_check_step_count(context: Context, count: int) -> None:
|
|
assert context.guardrails.step_count == count
|
|
|
|
|
|
@then("the guardrails budget_spent should be {amount:g}")
|
|
def step_check_budget_spent(context: Context, amount: float) -> None:
|
|
assert context.guardrails.budget_spent == amount
|
|
|
|
|
|
@then("the guardrails should have empty required_confirmations")
|
|
def step_check_empty_confirmations(context: Context) -> None:
|
|
assert context.guardrails.required_confirmations == []
|
|
|
|
|
|
@then("a guardrails validation error should be raised")
|
|
def step_check_validation_error(context: Context) -> None:
|
|
assert context.guardrails_error is not None
|
|
|
|
|
|
@then('the guardrails error should mention "{text}"')
|
|
def step_check_error_message(context: Context, text: str) -> None:
|
|
assert text in str(context.guardrails_error)
|
|
|
|
|
|
# ---- Step limit checks ----
|
|
|
|
|
|
@given("guardrails with max_steps {max_steps:d}")
|
|
def step_given_guardrails_max_steps(
|
|
context: Context,
|
|
max_steps: int,
|
|
) -> None:
|
|
context.guardrails = AutonomyGuardrails(max_steps=max_steps)
|
|
|
|
|
|
@given("guardrails with no step limit")
|
|
def step_given_no_step_limit(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when("I check step limit at step {step:d}")
|
|
def step_check_step_limit(context: Context, step: int) -> None:
|
|
context.guardrails.step_count = step
|
|
allowed, reason = context.guardrails.check_step_limit()
|
|
context.step_allowed = allowed
|
|
context.step_reason = reason
|
|
|
|
|
|
@then("the step check should be allowed")
|
|
def step_check_allowed(context: Context) -> None:
|
|
assert context.step_allowed is True
|
|
|
|
|
|
@then("the step check should be denied")
|
|
def step_check_denied(context: Context) -> None:
|
|
assert context.step_allowed is False
|
|
|
|
|
|
# ---- Budget checks ----
|
|
|
|
|
|
@given("guardrails with tool_budget {budget:g}")
|
|
def step_given_guardrails_budget(
|
|
context: Context,
|
|
budget: float,
|
|
) -> None:
|
|
context.guardrails = AutonomyGuardrails(tool_budget=budget)
|
|
|
|
|
|
@given("guardrails with tool_budget {budget:g} and budget_spent {spent:g}")
|
|
def step_given_guardrails_budget_spent(
|
|
context: Context,
|
|
budget: float,
|
|
spent: float,
|
|
) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
tool_budget=budget,
|
|
budget_spent=spent,
|
|
)
|
|
|
|
|
|
@given("guardrails with no budget limit")
|
|
def step_given_no_budget_limit(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when("I check tool budget for cost {cost:g}")
|
|
def step_check_budget(context: Context, cost: float) -> None:
|
|
allowed, reason = context.guardrails.check_tool_budget(cost)
|
|
context.budget_allowed = allowed
|
|
context.budget_reason = reason
|
|
# Provide a BudgetCheckResult for shared @then steps (#584)
|
|
from cleveragents.domain.models.core.cost_budget import BudgetCheckResult
|
|
|
|
context.budget_result = BudgetCheckResult(allowed=allowed, reason=reason or "")
|
|
|
|
|
|
@when("I try to check tool budget for cost {cost:g}")
|
|
def step_try_check_budget_negative(
|
|
context: Context,
|
|
cost: float,
|
|
) -> None:
|
|
try:
|
|
context.guardrails.check_tool_budget(cost)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
# "the budget check should be allowed/denied" steps moved to
|
|
# cost_budgets_steps.py (shared across both feature files, #584).
|
|
|
|
|
|
# ---- Confirmation checks ----
|
|
|
|
|
|
@given('guardrails with required confirmations "{confirmations}"')
|
|
def step_given_confirmations(
|
|
context: Context,
|
|
confirmations: str,
|
|
) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
required_confirmations=confirmations.split(","),
|
|
)
|
|
|
|
|
|
@given("guardrails with no required confirmations")
|
|
def step_given_no_confirmations(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when('I check confirmation for operation "{operation}"')
|
|
def step_check_confirmation(context: Context, operation: str) -> None:
|
|
required, reason = context.guardrails.check_confirmation_required(operation)
|
|
context.confirmation_required = required
|
|
context.confirmation_reason = reason
|
|
|
|
|
|
@then("confirmation should be required")
|
|
def step_confirmation_required(context: Context) -> None:
|
|
assert context.confirmation_required is True
|
|
|
|
|
|
@then("confirmation should not be required")
|
|
def step_confirmation_not_required(context: Context) -> None:
|
|
assert context.confirmation_required is False
|
|
|
|
|
|
# ---- Step counter and budget tracking ----
|
|
|
|
|
|
@when("I increment the step counter {count:d} times")
|
|
def step_increment_counter(context: Context, count: int) -> None:
|
|
for _ in range(count):
|
|
context.guardrails.increment_step()
|
|
|
|
|
|
@then("the step_count should be {count:d}")
|
|
def step_verify_step_count(context: Context, count: int) -> None:
|
|
assert context.guardrails.step_count == count
|
|
|
|
|
|
@when("I record a tool cost of {cost:g}")
|
|
def step_record_cost(context: Context, cost: float) -> None:
|
|
context.guardrails.record_cost(cost)
|
|
|
|
|
|
@then("the budget_spent should be {amount:g}")
|
|
def step_verify_budget_spent(context: Context, amount: float) -> None:
|
|
assert abs(context.guardrails.budget_spent - amount) < 0.001
|
|
|
|
|
|
@when("I try to record a negative cost")
|
|
def step_try_record_negative_cost(context: Context) -> None:
|
|
try:
|
|
context.guardrails.record_cost(-5.0)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
# ---- Audit trail ----
|
|
|
|
|
|
@given("an empty audit trail")
|
|
def step_given_empty_trail(context: Context) -> None:
|
|
context.audit_trail = GuardrailAuditTrail()
|
|
|
|
|
|
@when("I add a step_allowed entry to the audit trail")
|
|
def step_add_allowed_entry(context: Context) -> None:
|
|
entry = GuardrailAuditEntry(
|
|
event_type=GuardrailEventType.STEP_ALLOWED,
|
|
guard_name="step_limit",
|
|
result=GuardrailResult.ALLOWED,
|
|
)
|
|
context.audit_trail.add_entry(entry)
|
|
|
|
|
|
@when("I add a budget_blocked entry to the audit trail")
|
|
def step_add_blocked_entry(context: Context) -> None:
|
|
entry = GuardrailAuditEntry(
|
|
event_type=GuardrailEventType.BUDGET_BLOCKED,
|
|
guard_name="tool_budget",
|
|
result=GuardrailResult.DENIED,
|
|
reason="Budget exceeded",
|
|
)
|
|
context.audit_trail.add_entry(entry)
|
|
|
|
|
|
@then("the audit trail should have {count:d} entries")
|
|
def step_check_trail_count(context: Context, count: int) -> None:
|
|
trail = getattr(context, "audit_trail", None)
|
|
if trail is None:
|
|
trail = context.service_trail
|
|
assert len(trail.entries) == count
|
|
|
|
|
|
@then("the audit trail allowed_count should be {count:d}")
|
|
def step_check_allowed_count(context: Context, count: int) -> None:
|
|
assert context.audit_trail.allowed_count == count
|
|
|
|
|
|
@then("the audit trail denied_count should be {count:d}")
|
|
def step_check_denied_count(context: Context, count: int) -> None:
|
|
assert context.audit_trail.denied_count == count
|
|
|
|
|
|
@when(
|
|
"I create an audit entry with event_type {event_type} and guard_name {guard_name}"
|
|
)
|
|
def step_create_entry(
|
|
context: Context,
|
|
event_type: str,
|
|
guard_name: str,
|
|
) -> None:
|
|
context.audit_entry = GuardrailAuditEntry(
|
|
event_type=GuardrailEventType(event_type),
|
|
guard_name=guard_name,
|
|
result=GuardrailResult.DENIED,
|
|
reason="Test reason",
|
|
)
|
|
|
|
|
|
@then('the audit entry event_type should be "{value}"')
|
|
def step_check_entry_event_type(context: Context, value: str) -> None:
|
|
assert context.audit_entry.event_type.value == value
|
|
|
|
|
|
@then('the audit entry guard_name should be "{value}"')
|
|
def step_check_entry_guard_name(context: Context, value: str) -> None:
|
|
assert context.audit_entry.guard_name == value
|
|
|
|
|
|
@then('the audit entry result should be "{value}"')
|
|
def step_check_entry_result(context: Context, value: str) -> None:
|
|
assert context.audit_entry.result.value == value
|
|
|
|
|
|
@then("the audit entry should have a timestamp")
|
|
def step_check_entry_timestamp(context: Context) -> None:
|
|
assert context.audit_entry.timestamp is not None
|
|
assert len(context.audit_entry.timestamp) > 0
|
|
|
|
|
|
# ---- Service steps ----
|
|
|
|
|
|
@given("an autonomy guardrail service")
|
|
def step_given_service(context: Context) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
|
|
|
|
@when('I configure guardrails for plan "{plan_id}" with max_steps {max_steps:d}')
|
|
def step_configure_guardrails(
|
|
context: Context,
|
|
plan_id: str,
|
|
max_steps: int,
|
|
) -> None:
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(max_steps=max_steps),
|
|
)
|
|
|
|
|
|
@then('the service should return guardrails for plan "{plan_id}"')
|
|
def step_service_has_guardrails(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
g = context.guardrail_service.get_guardrails(plan_id)
|
|
assert g is not None
|
|
context.service_guardrails = g
|
|
|
|
|
|
@then('the service should return None for plan "{plan_id}"')
|
|
def step_service_no_guardrails(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
g = context.guardrail_service.get_guardrails(plan_id)
|
|
assert g is None
|
|
|
|
|
|
@then("the service guardrails should have max_steps {value:d}")
|
|
def step_service_max_steps(context: Context, value: int) -> None:
|
|
assert context.service_guardrails.max_steps == value
|
|
|
|
|
|
@given('an autonomy guardrail service with guardrails for plan "{plan_id}"')
|
|
def step_given_service_with_guardrails(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(max_steps=10),
|
|
)
|
|
|
|
|
|
@given('an autonomy guardrail service with budget guardrails for plan "{plan_id}"')
|
|
def step_given_service_with_budget(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(tool_budget=100.0),
|
|
)
|
|
|
|
|
|
@given(
|
|
'an autonomy guardrail service with confirmation guardrails for plan "{plan_id}"'
|
|
)
|
|
def step_given_service_with_confirmations(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(required_confirmations=["deploy", "delete"]),
|
|
)
|
|
|
|
|
|
@when('I check step limit via service for plan "{plan_id}" at step {step:d}')
|
|
def step_service_check_step(
|
|
context: Context,
|
|
plan_id: str,
|
|
step: int,
|
|
) -> None:
|
|
context.step_result = context.guardrail_service.check_step_limit(plan_id, step)
|
|
|
|
|
|
@when('I check tool budget via service for plan "{plan_id}" with cost {cost:g}')
|
|
def step_service_check_budget(
|
|
context: Context,
|
|
plan_id: str,
|
|
cost: float,
|
|
) -> None:
|
|
context.budget_result = context.guardrail_service.check_tool_budget(plan_id, cost)
|
|
|
|
|
|
@when('I check confirmation via service for plan "{plan_id}" operation "{operation}"')
|
|
def step_service_check_confirmation(
|
|
context: Context,
|
|
plan_id: str,
|
|
operation: str,
|
|
) -> None:
|
|
context.confirmation_result = context.guardrail_service.check_confirmation_required(
|
|
plan_id, operation
|
|
)
|
|
|
|
|
|
@then('the audit trail for plan "{plan_id}" should have {count:d} entry')
|
|
def step_service_trail_count(
|
|
context: Context,
|
|
plan_id: str,
|
|
count: int,
|
|
) -> None:
|
|
trail = context.guardrail_service.get_audit_trail(plan_id)
|
|
context.service_trail = trail
|
|
assert len(trail.entries) == count
|
|
|
|
|
|
@then("the service guardrails budget_spent should be {amount:g}")
|
|
def step_service_budget_spent(context: Context, amount: float) -> None:
|
|
# Use the plan_id stored by the service budget step (plan-003)
|
|
guardrails = context.guardrail_service.get_guardrails("plan-003")
|
|
assert guardrails is not None, "No budget guardrails found for plan-003"
|
|
assert abs(guardrails.budget_spent - amount) < 0.001
|
|
|
|
|
|
@then('the metadata for plan "{plan_id}" should contain the audit trail')
|
|
def step_service_metadata(context: Context, plan_id: str) -> None:
|
|
metadata = context.guardrail_service.get_audit_trail_as_metadata(plan_id)
|
|
assert "guardrail_audit_trail" in metadata
|
|
assert "autonomy_guardrails" in metadata
|
|
|
|
|
|
@when('I load guardrails from metadata for plan "{plan_id}"')
|
|
def step_load_metadata(context: Context, plan_id: str) -> None:
|
|
metadata = {
|
|
"autonomy_guardrails": {
|
|
"max_steps": 20,
|
|
"tool_budget": 50.0,
|
|
"required_confirmations": [],
|
|
"step_count": 0,
|
|
"budget_spent": 0.0,
|
|
},
|
|
"guardrail_audit_trail": {"entries": []},
|
|
}
|
|
context.guardrail_service.load_from_metadata(plan_id, metadata)
|
|
|
|
|
|
@when('I remove plan "{plan_id}" from the service')
|
|
def step_remove_plan(context: Context, plan_id: str) -> None:
|
|
context.guardrail_service.remove_plan(plan_id)
|
|
|
|
|
|
@then("the step limit check should return True")
|
|
def step_check_returns_true(context: Context) -> None:
|
|
assert context.step_result is True
|
|
|
|
|
|
@then("the tool budget check should return True")
|
|
def step_budget_returns_true(context: Context) -> None:
|
|
assert context.budget_result is True
|
|
|
|
|
|
@then("the confirmation check should return False")
|
|
def step_confirmation_returns_false(context: Context) -> None:
|
|
assert context.confirmation_result is False
|
|
|
|
|
|
@when("I try to configure guardrails with empty plan_id")
|
|
def step_try_empty_plan_id(context: Context) -> None:
|
|
try:
|
|
context.guardrail_service.configure_guardrails(
|
|
"",
|
|
AutonomyGuardrails(),
|
|
)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@then("GuardrailEventType should have at least {count:d} members")
|
|
def step_check_event_type_members(
|
|
context: Context,
|
|
count: int,
|
|
) -> None:
|
|
assert len(GuardrailEventType) >= count
|
|
|
|
|
|
@then("GuardrailResult should have exactly {count:d} members")
|
|
def step_check_result_members(context: Context, count: int) -> None:
|
|
assert len(GuardrailResult) == count
|
|
|
|
|
|
# ---- Wall-clock time steps ----
|
|
|
|
|
|
@given("guardrails with no wall-clock limit")
|
|
def step_given_no_wall_clock(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@given("guardrails with wall-clock limit {seconds:g} seconds started recently")
|
|
def step_given_wall_clock_recent(context: Context, seconds: float) -> None:
|
|
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=seconds)
|
|
context.guardrails.mark_started()
|
|
|
|
|
|
@given("guardrails with wall-clock limit that has already expired")
|
|
def step_given_wall_clock_expired(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=0.001)
|
|
# Set start_time far in the past to guarantee expiry
|
|
context.guardrails.start_time = (
|
|
datetime.now(tz=UTC) - timedelta(seconds=10)
|
|
).isoformat()
|
|
|
|
|
|
@when("I check wall-clock time")
|
|
def step_check_wall_clock(context: Context) -> None:
|
|
allowed, reason = context.guardrails.check_wall_clock()
|
|
context.wall_clock_allowed = allowed
|
|
context.wall_clock_reason = reason
|
|
|
|
|
|
@then("the wall-clock check should be allowed")
|
|
def step_wall_clock_allowed(context: Context) -> None:
|
|
assert context.wall_clock_allowed is True
|
|
|
|
|
|
@then("the wall-clock check should be denied")
|
|
def step_wall_clock_denied(context: Context) -> None:
|
|
assert context.wall_clock_allowed is False
|
|
|
|
|
|
@given('an autonomy guardrail service with wall-clock guardrails for plan "{plan_id}"')
|
|
def step_given_service_wall_clock(context: Context, plan_id: str) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
g = AutonomyGuardrails(max_wall_clock_seconds=600.0)
|
|
g.mark_started()
|
|
context.guardrail_service.configure_guardrails(plan_id, g)
|
|
|
|
|
|
@when('I check wall-clock via service for plan "{plan_id}"')
|
|
def step_service_check_wall_clock(context: Context, plan_id: str) -> None:
|
|
context.wall_clock_result = context.guardrail_service.check_wall_clock(plan_id)
|
|
|
|
|
|
# ---- Actor tool-call limit steps ----
|
|
|
|
|
|
@given("guardrails with actor tool-call limit {limit:d}")
|
|
def step_given_actor_limit(context: Context, limit: int) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
actor_limits=ActorLimits(max_tool_calls_per_invocation=limit),
|
|
)
|
|
|
|
|
|
@given("guardrails with no actor tool-call limit")
|
|
def step_given_no_actor_limit(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when("I check actor tool calls with {calls:d} current calls")
|
|
def step_check_actor_calls(context: Context, calls: int) -> None:
|
|
allowed, reason = context.guardrails.check_actor_tool_calls(calls)
|
|
context.actor_allowed = allowed
|
|
context.actor_reason = reason
|
|
|
|
|
|
@then("the actor tool-call check should be allowed")
|
|
def step_actor_allowed(context: Context) -> None:
|
|
assert context.actor_allowed is True
|
|
|
|
|
|
@then("the actor tool-call check should be denied")
|
|
def step_actor_denied(context: Context) -> None:
|
|
assert context.actor_allowed is False
|
|
|
|
|
|
@given('an autonomy guardrail service with actor-limit guardrails for plan "{plan_id}"')
|
|
def step_given_service_actor_limit(context: Context, plan_id: str) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(
|
|
actor_limits=ActorLimits(max_tool_calls_per_invocation=10),
|
|
),
|
|
)
|
|
|
|
|
|
@when('I check actor tool calls via service for plan "{plan_id}" with {calls:d} calls')
|
|
def step_service_check_actor_calls(
|
|
context: Context,
|
|
plan_id: str,
|
|
calls: int,
|
|
) -> None:
|
|
context.actor_result = context.guardrail_service.check_actor_tool_calls(
|
|
plan_id, calls
|
|
)
|
|
|
|
|
|
# ---- Bounded audit trail eviction steps ----
|
|
|
|
|
|
@given("an audit trail with max_entries {max_entries:d}")
|
|
def step_given_bounded_trail(context: Context, max_entries: int) -> None:
|
|
context.audit_trail = GuardrailAuditTrail(max_entries=max_entries)
|
|
|
|
|
|
@when("I add {count:d} step_allowed entries to the audit trail")
|
|
def step_add_many_allowed_entries(context: Context, count: int) -> None:
|
|
for _ in range(count):
|
|
entry = GuardrailAuditEntry(
|
|
event_type=GuardrailEventType.STEP_ALLOWED,
|
|
guard_name="step_limit",
|
|
result=GuardrailResult.ALLOWED,
|
|
)
|
|
context.audit_trail.add_entry(entry)
|
|
|
|
|
|
# ---- Edge-case steps ----
|
|
|
|
|
|
@when('I try to load metadata with oversized audit trail for plan "{plan_id}"')
|
|
def step_try_load_oversized_trail(context: Context, plan_id: str) -> None:
|
|
metadata = {
|
|
"guardrail_audit_trail": {
|
|
"entries": [{}] * (_MAX_METADATA_ENTRIES + 1),
|
|
},
|
|
}
|
|
try:
|
|
context.guardrail_service.load_from_metadata(plan_id, metadata)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@when('I try to load metadata with oversized confirmations for plan "{plan_id}"')
|
|
def step_try_load_oversized_confirmations(
|
|
context: Context,
|
|
plan_id: str,
|
|
) -> None:
|
|
metadata = {
|
|
"autonomy_guardrails": {
|
|
"max_steps": 10,
|
|
"tool_budget": 50.0,
|
|
"required_confirmations": ["op"] * (_MAX_CONFIRMATIONS + 1),
|
|
"step_count": 0,
|
|
"budget_spent": 0.0,
|
|
},
|
|
}
|
|
try:
|
|
context.guardrail_service.load_from_metadata(plan_id, metadata)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@when('I serialize and restore metadata for plan "{plan_id}"')
|
|
def step_round_trip_metadata(context: Context, plan_id: str) -> None:
|
|
metadata = context.guardrail_service.get_audit_trail_as_metadata(plan_id)
|
|
context.round_trip_metadata = metadata
|
|
# Create a fresh service and restore
|
|
context.restored_service = AutonomyGuardrailService()
|
|
context.restored_service.load_from_metadata(plan_id, metadata)
|
|
|
|
|
|
@then('the restored guardrails for plan "{plan_id}" should match the original')
|
|
def step_verify_round_trip(context: Context, plan_id: str) -> None:
|
|
original = context.guardrail_service.get_guardrails(plan_id)
|
|
restored = context.restored_service.get_guardrails(plan_id)
|
|
assert original is not None
|
|
assert restored is not None
|
|
assert restored.max_steps == original.max_steps
|
|
assert restored.tool_budget == original.tool_budget
|
|
assert restored.required_confirmations == original.required_confirmations
|
|
assert restored.step_count == original.step_count
|
|
# Check audit trail was also restored
|
|
original_trail = context.guardrail_service.get_audit_trail(plan_id)
|
|
restored_trail = context.restored_service.get_audit_trail(plan_id)
|
|
assert len(restored_trail.entries) == len(original_trail.entries)
|
|
|
|
|
|
# ---- start_time validation steps ----
|
|
|
|
|
|
@when("I create guardrails with a valid start_time")
|
|
def step_create_guardrails_valid_start_time(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
start_time=datetime.now(tz=UTC).isoformat(),
|
|
)
|
|
|
|
|
|
@when('I try to create guardrails with start_time "{value}"')
|
|
def step_try_create_bad_start_time(context: Context, value: str) -> None:
|
|
try:
|
|
AutonomyGuardrails(start_time=value)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@then("the guardrails start_time should be set")
|
|
def step_check_start_time_set(context: Context) -> None:
|
|
assert context.guardrails.start_time is not None
|
|
assert len(context.guardrails.start_time) > 0
|
|
|
|
|
|
@then("the guardrails start_time should be None")
|
|
def step_check_start_time_none(context: Context) -> None:
|
|
assert context.guardrails.start_time is None
|
|
|
|
|
|
@given("guardrails with a malformed start_time bypassing validation")
|
|
def step_given_malformed_start_time(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails(max_wall_clock_seconds=60.0)
|
|
# Bypass Pydantic validation by setting directly via object.__setattr__
|
|
object.__setattr__(context.guardrails, "start_time", "not-a-date")
|
|
|
|
|
|
# ---- Retry-per-failure check steps ----
|
|
|
|
|
|
@given("guardrails with retry limit {limit:d}")
|
|
def step_given_retry_limit(context: Context, limit: int) -> None:
|
|
context.guardrails = AutonomyGuardrails(
|
|
actor_limits=ActorLimits(max_retries_per_failure=limit),
|
|
)
|
|
|
|
|
|
@given("guardrails with no retry limit")
|
|
def step_given_no_retry_limit(context: Context) -> None:
|
|
context.guardrails = AutonomyGuardrails()
|
|
|
|
|
|
@when("I check retries per failure with {retries:d} current retries")
|
|
def step_check_retries(context: Context, retries: int) -> None:
|
|
allowed, reason = context.guardrails.check_retries_per_failure(retries)
|
|
context.retry_allowed = allowed
|
|
context.retry_reason = reason
|
|
|
|
|
|
@then("the retry check should be allowed")
|
|
def step_retry_allowed(context: Context) -> None:
|
|
assert context.retry_allowed is True
|
|
|
|
|
|
@then("the retry check should be denied")
|
|
def step_retry_denied(context: Context) -> None:
|
|
assert context.retry_allowed is False
|
|
|
|
|
|
@given('an autonomy guardrail service with retry-limit guardrails for plan "{plan_id}"')
|
|
def step_given_service_retry_limit(context: Context, plan_id: str) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(
|
|
actor_limits=ActorLimits(max_retries_per_failure=3),
|
|
),
|
|
)
|
|
|
|
|
|
@when('I check retries via service for plan "{plan_id}" with {retries:d} retries')
|
|
def step_service_check_retries(
|
|
context: Context,
|
|
plan_id: str,
|
|
retries: int,
|
|
) -> None:
|
|
context.retry_result = context.guardrail_service.check_retries_per_failure(
|
|
plan_id, retries
|
|
)
|
|
|
|
|
|
@then("the retry service check should return False")
|
|
def step_retry_service_returns_false(context: Context) -> None:
|
|
assert context.retry_result is False
|
|
|
|
|
|
@given(
|
|
'a guardrail service with max_retries_per_failure {limit:d} for plan "{plan_id}"'
|
|
)
|
|
def step_given_service_custom_retry_limit(
|
|
context: Context, limit: int, plan_id: str
|
|
) -> None:
|
|
context.guardrail_service = AutonomyGuardrailService()
|
|
context.guardrail_service.configure_guardrails(
|
|
plan_id,
|
|
AutonomyGuardrails(
|
|
actor_limits=ActorLimits(max_retries_per_failure=limit),
|
|
),
|
|
)
|
|
|
|
|
|
@when('I check retries for plan "{plan_id}" incrementing from {start:d} to {end:d}')
|
|
def step_check_retries_incrementing(
|
|
context: Context, plan_id: str, start: int, end: int
|
|
) -> None:
|
|
context.retry_results = []
|
|
for i in range(start, end + 1):
|
|
allowed = context.guardrail_service.check_retries_per_failure(plan_id, i)
|
|
context.retry_results.append((i, allowed))
|
|
|
|
|
|
@then("retry checks {allowed_csv} should be allowed")
|
|
def step_retry_checks_allowed(context: Context, allowed_csv: str) -> None:
|
|
expected = [int(x.strip()) for x in allowed_csv.split(",")]
|
|
results_map = dict(context.retry_results)
|
|
for idx in expected:
|
|
assert results_map[idx] is True, (
|
|
f"Expected retry {idx} to be allowed, but it was denied"
|
|
)
|
|
|
|
|
|
@then("retry checks {denied_csv} should be denied")
|
|
def step_retry_checks_denied(context: Context, denied_csv: str) -> None:
|
|
expected = [int(x.strip()) for x in denied_csv.split(",")]
|
|
results_map = dict(context.retry_results)
|
|
for idx in expected:
|
|
assert results_map[idx] is False, (
|
|
f"Expected retry {idx} to be denied, but it was allowed"
|
|
)
|
|
|
|
|
|
# ---- PlanExecutor guardrail integration helpers ----
|
|
|
|
|
|
def _make_guardrail_mock_plan(
|
|
phase: PlanPhase = PlanPhase.EXECUTE,
|
|
state: ProcessingState = ProcessingState.QUEUED,
|
|
definition_of_done: str = "Step A\nStep B",
|
|
decision_root_id: str = "root-001",
|
|
) -> MagicMock:
|
|
"""Create a mock plan suitable for PlanExecutor tests."""
|
|
plan = MagicMock()
|
|
plan.phase = phase
|
|
plan.state = state
|
|
plan.definition_of_done = definition_of_done
|
|
plan.decision_root_id = decision_root_id
|
|
plan.timestamps = PlanTimestamps()
|
|
plan.error_details = {}
|
|
plan.changeset_id = None
|
|
plan.sandbox_refs = []
|
|
plan.read_only = False
|
|
plan.invariants = []
|
|
return plan
|
|
|
|
|
|
def _make_guardrail_mock_lifecycle(plan: MagicMock) -> MagicMock:
|
|
"""Create a mock lifecycle service for PlanExecutor tests."""
|
|
lcs = MagicMock()
|
|
lcs.get_plan.return_value = plan
|
|
lcs.start_strategize = MagicMock()
|
|
lcs.complete_strategize = MagicMock()
|
|
lcs.fail_strategize = MagicMock()
|
|
lcs.start_execute = MagicMock()
|
|
lcs.complete_execute = MagicMock()
|
|
lcs.fail_execute = MagicMock()
|
|
lcs.commit_plan = MagicMock()
|
|
return lcs
|
|
|
|
|
|
# ---- PlanExecutor guardrail integration steps ----
|
|
|
|
|
|
@given("a plan executor with guardrail service and step limit {limit:d}")
|
|
def step_given_executor_with_step_limit(context: Context, limit: int) -> None:
|
|
service = AutonomyGuardrailService()
|
|
service.configure_guardrails(
|
|
"grd-plan-001",
|
|
AutonomyGuardrails(max_steps=limit),
|
|
)
|
|
context.guardrail_plan_id = "grd-plan-001"
|
|
context.guardrail_exec_service = service
|
|
|
|
|
|
@given("a plan executor with guardrail service and expired wall-clock")
|
|
def step_given_executor_with_expired_wall_clock(context: Context) -> None:
|
|
service = AutonomyGuardrailService()
|
|
g = AutonomyGuardrails(max_wall_clock_seconds=0.001)
|
|
g.start_time = (datetime.now(tz=UTC) - timedelta(seconds=10)).isoformat()
|
|
service.configure_guardrails("grd-plan-002", g)
|
|
context.guardrail_plan_id = "grd-plan-002"
|
|
context.guardrail_exec_service = service
|
|
|
|
|
|
@given("a plan executor without guardrail service")
|
|
def step_given_executor_no_guardrails(context: Context) -> None:
|
|
context.guardrail_plan_id = "grd-plan-003"
|
|
context.guardrail_exec_service = None
|
|
|
|
|
|
@given("a plan executor with guardrail service and no wall-clock limit")
|
|
def step_given_executor_no_wall_clock(context: Context) -> None:
|
|
service = AutonomyGuardrailService()
|
|
service.configure_guardrails(
|
|
"grd-plan-004",
|
|
AutonomyGuardrails(max_steps=100),
|
|
)
|
|
context.guardrail_plan_id = "grd-plan-004"
|
|
context.guardrail_exec_service = service
|
|
|
|
|
|
@given("a plan in execute phase with {count:d} decisions")
|
|
def step_given_plan_in_execute_phase(context: Context, count: int) -> None:
|
|
steps = "\n".join(f"Step {chr(65 + i)}" for i in range(count))
|
|
plan = _make_guardrail_mock_plan(definition_of_done=steps)
|
|
lifecycle = _make_guardrail_mock_lifecycle(plan)
|
|
context.guardrail_executor = PlanExecutor(
|
|
lifecycle_service=lifecycle,
|
|
guardrail_service=context.guardrail_exec_service,
|
|
)
|
|
|
|
|
|
@when("I run execute on the plan")
|
|
def step_run_execute(context: Context) -> None:
|
|
try:
|
|
context.execute_result = context.guardrail_executor.run_execute(
|
|
context.guardrail_plan_id,
|
|
)
|
|
context.execute_error = None
|
|
except Exception as exc:
|
|
context.execute_error = exc
|
|
context.execute_result = None
|
|
|
|
|
|
@then("execution should fail with a guardrail step limit error")
|
|
def step_check_step_limit_error(context: Context) -> None:
|
|
assert context.execute_error is not None
|
|
assert isinstance(context.execute_error, PlanError)
|
|
assert "step limit" in str(context.execute_error).lower()
|
|
|
|
|
|
@then("execution should fail with a guardrail wall-clock error")
|
|
def step_check_wall_clock_error(context: Context) -> None:
|
|
assert context.execute_error is not None
|
|
assert isinstance(context.execute_error, PlanError)
|
|
assert "wall-clock" in str(context.execute_error).lower()
|
|
|
|
|
|
@then("execution should succeed")
|
|
def step_check_execution_success(context: Context) -> None:
|
|
assert context.execute_error is None
|
|
assert context.execute_result is not None
|
|
|
|
|
|
@then("the guardrails start_time should have been set")
|
|
def step_check_start_time_was_set(context: Context) -> None:
|
|
service = context.guardrail_exec_service
|
|
assert service is not None
|
|
guardrails = service.get_guardrails(context.guardrail_plan_id)
|
|
assert guardrails is not None
|
|
assert guardrails.start_time is not None
|
|
|
|
|
|
# ---- Additional validation edge-case steps ----
|
|
|
|
|
|
@when("I try to create actor limits with max_tool_calls_per_invocation {value:d}")
|
|
def step_try_bad_max_tool_calls(context: Context, value: int) -> None:
|
|
try:
|
|
ActorLimits(max_tool_calls_per_invocation=value)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@when("I try to create actor limits with max_retries_per_failure {value:d}")
|
|
def step_try_bad_max_retries(context: Context, value: int) -> None:
|
|
try:
|
|
ActorLimits(max_retries_per_failure=value)
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|
|
|
|
|
|
@when("I try to create guardrails with max_wall_clock_seconds {value:d}")
|
|
def step_try_bad_wall_clock(context: Context, value: int) -> None:
|
|
try:
|
|
AutonomyGuardrails(max_wall_clock_seconds=float(value))
|
|
context.guardrails_error = None
|
|
except Exception as exc:
|
|
context.guardrails_error = exc
|