523 lines
20 KiB
Python
523 lines
20 KiB
Python
"""Step definitions for Automation Profile domain model tests."""
|
|
|
|
from typing import Any
|
|
|
|
from behave import then, when
|
|
from behave.runner import Context
|
|
from pydantic import ValidationError
|
|
|
|
from cleveragents.domain.models.core.automation_profile import (
|
|
BUILTIN_PROFILES,
|
|
AutomationProfile,
|
|
get_builtin_profile,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_profile(**overrides: Any) -> AutomationProfile:
|
|
"""Create a profile with sensible defaults."""
|
|
defaults: dict[str, Any] = {
|
|
"name": "test/default",
|
|
}
|
|
defaults.update(overrides)
|
|
return AutomationProfile(**defaults)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Profile creation with specific thresholds
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I create a profile with auto_strategize {value:g}")
|
|
def step_create_profile_auto_strategize(context: Context, value: float) -> None:
|
|
"""Create a profile with a specific auto_strategize."""
|
|
context.profile_model = _make_profile(auto_strategize=value)
|
|
context.profile_error = None
|
|
|
|
|
|
@when("I try to create a profile with auto_strategize {value:g}")
|
|
def step_try_create_profile_auto_strategize(context: Context, value: float) -> None:
|
|
"""Try creating a profile with invalid auto_strategize."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_strategize=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_execute {value:g}")
|
|
def step_try_create_profile_auto_execute(context: Context, value: float) -> None:
|
|
"""Try creating a profile with invalid auto_execute."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_execute=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_apply {value:g}")
|
|
def step_try_create_profile_auto_apply(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_apply."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_apply=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_decisions_strategize {value:g}")
|
|
def step_try_create_profile_auto_dec_strat(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_decisions_strategize."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_decisions_strategize=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_decisions_execute {value:g}")
|
|
def step_try_create_profile_auto_dec_exec(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_decisions_execute."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_decisions_execute=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_validation_fix {value:g}")
|
|
def step_try_create_profile_auto_val_fix(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_validation_fix."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_validation_fix=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_strategy_revision {value:g}")
|
|
def step_try_create_profile_auto_strat_rev(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_strategy_revision."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_strategy_revision=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_reversion_from_apply {value:g}")
|
|
def step_try_create_profile_auto_rev_apply(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_reversion_from_apply."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_reversion_from_apply=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_child_plans {value:g}")
|
|
def step_try_create_profile_auto_child(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_child_plans."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_child_plans=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_retry_transient {value:g}")
|
|
def step_try_create_profile_auto_retry(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_retry_transient."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_retry_transient=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with auto_checkpoint_restore {value:g}")
|
|
def step_try_create_profile_auto_ckpt(context: Context, value: float) -> None:
|
|
"""Try creating with invalid auto_checkpoint_restore."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = _make_profile(auto_checkpoint_restore=value)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Profile assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the profile model should be created")
|
|
def step_check_profile_created(context: Context) -> None:
|
|
"""Verify the profile was created."""
|
|
assert context.profile_model is not None, "Profile should be created"
|
|
|
|
|
|
@then("the profile auto_strategize should be {expected:g}")
|
|
def step_check_auto_strategize(context: Context, expected: float) -> None:
|
|
"""Check auto_strategize value."""
|
|
actual = context.profile_model.auto_strategize
|
|
assert actual == expected, f"Expected auto_strategize {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_execute should be {expected:g}")
|
|
def step_check_auto_execute(context: Context, expected: float) -> None:
|
|
"""Check auto_execute value."""
|
|
actual = context.profile_model.auto_execute
|
|
assert actual == expected, f"Expected auto_execute {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_apply should be {expected:g}")
|
|
def step_check_auto_apply(context: Context, expected: float) -> None:
|
|
"""Check auto_apply value."""
|
|
actual = context.profile_model.auto_apply
|
|
assert actual == expected, f"Expected auto_apply {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_decisions_strategize should be {expected:g}")
|
|
def step_check_auto_dec_strat(context: Context, expected: float) -> None:
|
|
"""Check auto_decisions_strategize value."""
|
|
actual = context.profile_model.auto_decisions_strategize
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_decisions_execute should be {expected:g}")
|
|
def step_check_auto_dec_exec(context: Context, expected: float) -> None:
|
|
"""Check auto_decisions_execute value."""
|
|
actual = context.profile_model.auto_decisions_execute
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_validation_fix should be {expected:g}")
|
|
def step_check_auto_val_fix(context: Context, expected: float) -> None:
|
|
"""Check auto_validation_fix value."""
|
|
actual = context.profile_model.auto_validation_fix
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_strategy_revision should be {expected:g}")
|
|
def step_check_auto_strat_rev(context: Context, expected: float) -> None:
|
|
"""Check auto_strategy_revision value."""
|
|
actual = context.profile_model.auto_strategy_revision
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_reversion_from_apply should be {expected:g}")
|
|
def step_check_auto_rev_apply(context: Context, expected: float) -> None:
|
|
"""Check auto_reversion_from_apply value."""
|
|
actual = context.profile_model.auto_reversion_from_apply
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_child_plans should be {expected:g}")
|
|
def step_check_auto_child(context: Context, expected: float) -> None:
|
|
"""Check auto_child_plans value."""
|
|
actual = context.profile_model.auto_child_plans
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_retry_transient should be {expected:g}")
|
|
def step_check_auto_retry(context: Context, expected: float) -> None:
|
|
"""Check auto_retry_transient value."""
|
|
actual = context.profile_model.auto_retry_transient
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile auto_checkpoint_restore should be {expected:g}")
|
|
def step_check_auto_ckpt(context: Context, expected: float) -> None:
|
|
"""Check auto_checkpoint_restore value."""
|
|
actual = context.profile_model.auto_checkpoint_restore
|
|
assert actual == expected, f"Expected {expected}, got {actual}"
|
|
|
|
|
|
@then("the profile require_sandbox should be {expected}")
|
|
def step_check_require_sandbox(context: Context, expected: str) -> None:
|
|
"""Check require_sandbox value."""
|
|
exp_bool = expected.lower() == "true"
|
|
actual = context.profile_model.require_sandbox
|
|
assert actual is exp_bool, f"Expected require_sandbox {exp_bool}, got {actual}"
|
|
|
|
|
|
@then("the profile require_checkpoints should be {expected}")
|
|
def step_check_require_checkpoints(context: Context, expected: str) -> None:
|
|
"""Check require_checkpoints value."""
|
|
exp_bool = expected.lower() == "true"
|
|
actual = context.profile_model.require_checkpoints
|
|
assert actual is exp_bool, f"Expected require_checkpoints {exp_bool}, got {actual}"
|
|
|
|
|
|
@then("the profile allow_unsafe_tools should be {expected}")
|
|
def step_check_allow_unsafe(context: Context, expected: str) -> None:
|
|
"""Check allow_unsafe_tools value."""
|
|
exp_bool = expected.lower() == "true"
|
|
actual = context.profile_model.allow_unsafe_tools
|
|
assert actual is exp_bool, f"Expected allow_unsafe_tools {exp_bool}, got {actual}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation errors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("a profile validation error should be raised")
|
|
def step_check_profile_validation_error(
|
|
context: Context,
|
|
) -> None:
|
|
"""Verify that a validation error was raised."""
|
|
assert context.profile_error is not None, "Expected a validation error to be raised"
|
|
|
|
|
|
@then('the profile error should mention "{text}"')
|
|
def step_check_profile_error_message(context: Context, text: str) -> None:
|
|
"""Check the error message contains expected text."""
|
|
error_str = str(context.profile_error)
|
|
assert text in error_str, f"Expected error to mention '{text}', got: {error_str}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Built-in profiles
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I load the built-in profile "{name}"')
|
|
def step_load_builtin_profile(context: Context, name: str) -> None:
|
|
"""Load a built-in profile by name."""
|
|
context.profile_model = get_builtin_profile(name)
|
|
context.profile_error = None
|
|
|
|
|
|
@then("there should be {count:d} built-in profiles")
|
|
def step_check_builtin_count(context: Context, count: int) -> None:
|
|
"""Check the number of built-in profiles."""
|
|
actual = len(BUILTIN_PROFILES)
|
|
assert actual == count, f"Expected {count} built-in profiles, got {actual}"
|
|
|
|
|
|
@then('built-in profile "{name}" should exist')
|
|
def step_check_builtin_exists(context: Context, name: str) -> None:
|
|
"""Check a built-in profile exists."""
|
|
assert name in BUILTIN_PROFILES, f"Expected built-in profile '{name}' to exist"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Custom profile from config
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I load a profile from config with name "{name}" and auto_apply {value:g}')
|
|
def step_load_profile_from_config(context: Context, name: str, value: float) -> None:
|
|
"""Load a profile from config dict."""
|
|
config = {
|
|
"name": name,
|
|
"description": "Custom profile",
|
|
"auto_apply": value,
|
|
}
|
|
context.profile_model = AutomationProfile.from_config(config)
|
|
context.profile_error = None
|
|
|
|
|
|
@when("I try to load a profile from config missing name")
|
|
def step_try_load_profile_config_missing_name(
|
|
context: Context,
|
|
) -> None:
|
|
"""Try loading profile config without name."""
|
|
context.profile_config_error = None
|
|
try:
|
|
AutomationProfile.from_config({"description": "No name"})
|
|
except ValueError as e:
|
|
context.profile_config_error = e
|
|
|
|
|
|
@then('a profile config error should be raised with "{field}"')
|
|
def step_check_profile_config_error(context: Context, field: str) -> None:
|
|
"""Check config error mentions field."""
|
|
assert context.profile_config_error is not None, (
|
|
f"Expected ValueError for missing '{field}'"
|
|
)
|
|
assert field in str(context.profile_config_error), (
|
|
f"Expected error to mention '{field}', got: {context.profile_config_error}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Name validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I create a profile with name "{name}"')
|
|
def step_create_profile_by_name(context: Context, name: str) -> None:
|
|
"""Create a profile with specific name."""
|
|
context.profile_model = AutomationProfile(name=name)
|
|
context.profile_error = None
|
|
|
|
|
|
@when('I try to create a profile with invalid name "{name}"')
|
|
def step_try_create_profile_invalid_name(context: Context, name: str) -> None:
|
|
"""Attempt to create a profile with invalid name."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = AutomationProfile(name=name)
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@when("I try to create a profile with empty name")
|
|
def step_try_create_profile_empty_name(
|
|
context: Context,
|
|
) -> None:
|
|
"""Attempt to create a profile with empty name."""
|
|
context.profile_error = None
|
|
context.profile_model = None
|
|
try:
|
|
context.profile_model = AutomationProfile(name="")
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
@then('the profile name should be "{expected}"')
|
|
def step_check_profile_name(context: Context, expected: str) -> None:
|
|
"""Check profile name."""
|
|
assert context.profile_model.name == expected, (
|
|
f"Expected name '{expected}', got '{context.profile_model.name}'"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# get_builtin_profile helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I call get_builtin_profile with "{name}"')
|
|
def step_call_get_builtin(context: Context, name: str) -> None:
|
|
"""Call get_builtin_profile."""
|
|
context.profile_model = get_builtin_profile(name)
|
|
context.profile_error = None
|
|
|
|
|
|
@when('I try to call get_builtin_profile with "{name}"')
|
|
def step_try_call_get_builtin(context: Context, name: str) -> None:
|
|
"""Try calling get_builtin_profile with bad name."""
|
|
context.profile_key_error = None
|
|
try:
|
|
get_builtin_profile(name)
|
|
except KeyError as e:
|
|
context.profile_key_error = e
|
|
|
|
|
|
@then("a profile key error should be raised")
|
|
def step_check_profile_key_error(
|
|
context: Context,
|
|
) -> None:
|
|
"""Verify KeyError was raised."""
|
|
assert context.profile_key_error is not None, "Expected a KeyError to be raised"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Schema version
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('the profile schema_version should be "{expected}"')
|
|
def step_check_schema_version(context: Context, expected: str) -> None:
|
|
"""Check profile schema_version."""
|
|
actual = context.profile_model.schema_version
|
|
assert actual == expected, f"Expected schema_version '{expected}', got '{actual}'"
|
|
|
|
|
|
@when('I create a profile with schema_version "{version}"')
|
|
def step_create_profile_with_version(context: Context, version: str) -> None:
|
|
"""Create a profile with custom schema_version."""
|
|
context.profile_model = AutomationProfile(
|
|
name="test-profile", schema_version=version
|
|
)
|
|
context.profile_error = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Description
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('the profile description should be "{expected}"')
|
|
def step_check_profile_description(context: Context, expected: str) -> None:
|
|
"""Check profile description."""
|
|
actual = context.profile_model.description
|
|
assert actual == expected, f"Expected description '{expected}', got '{actual}'"
|
|
|
|
|
|
@then("the profile description should be empty")
|
|
def step_check_profile_description_empty(
|
|
context: Context,
|
|
) -> None:
|
|
"""Check profile description is empty."""
|
|
actual = context.profile_model.description
|
|
assert actual == "", f"Expected empty description, got '{actual}'"
|
|
|
|
|
|
@when('I create a profile with description "{desc}"')
|
|
def step_create_profile_with_desc(context: Context, desc: str) -> None:
|
|
"""Create a profile with custom description."""
|
|
context.profile_model = AutomationProfile(name="desc-test", description=desc)
|
|
context.profile_error = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_assignment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I try to assign auto_strategize {value:g} on the profile")
|
|
def step_try_assign_threshold(context: Context, value: float) -> None:
|
|
"""Try to assign an invalid threshold on existing profile."""
|
|
context.profile_error = None
|
|
try:
|
|
context.profile_model.auto_strategize = value
|
|
except ValidationError as e:
|
|
context.profile_error = e
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# model_dump
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I create a profile and dump it with auto_apply {value:g}")
|
|
def step_create_profile_with_auto_apply(context: Context, value: float) -> None:
|
|
"""Create a profile and store model dump."""
|
|
context.profile_model = AutomationProfile(name="dump-test", auto_apply=value)
|
|
context.profile_dump = context.profile_model.model_dump()
|
|
context.profile_error = None
|
|
|
|
|
|
@then('the profile model dump should have key "{key}"')
|
|
def step_check_dump_key(context: Context, key: str) -> None:
|
|
"""Check profile model dump has key."""
|
|
assert key in context.profile_dump, (
|
|
f"Expected key '{key}' in model dump, keys: {list(context.profile_dump)}"
|
|
)
|
|
|
|
|
|
@then("the profile model dump auto_apply should be {expected:g}")
|
|
def step_check_dump_auto_apply(context: Context, expected: float) -> None:
|
|
"""Check auto_apply in model dump."""
|
|
actual = context.profile_dump["auto_apply"]
|
|
assert actual == expected, f"Expected auto_apply {expected}, got {actual}"
|