cleveragents-core/features/steps/automation_profiles_guards_steps.py

"""Step definitions for Automation Profile Guards tests."""

import tempfile
from typing import Any

import yaml
from behave import given, then, when
from behave.runner import Context
from pydantic import ValidationError

from cleveragents.application.services.automation_profile_service import (
    AutomationProfileService,
)
from cleveragents.core.exceptions import ValidationError as CAValidationError
from cleveragents.domain.models.core.automation_guard import (
    AutomationGuard,
    GuardResult,
    GuardScope,
)
from cleveragents.domain.models.core.automation_profile import (
    AutomationProfile,
)


def _make_guarded_profile(**guard_kwargs: Any) -> AutomationProfile:
    """Create a profile with specific guard settings."""
    guards = AutomationGuard(**guard_kwargs)
    return AutomationProfile(name="test/guarded", guards=guards)


# -- Guard setup steps --


@given("a profile with max_tool_calls_per_step {limit:d}")
def step_profile_max_calls(context: Context, limit: int) -> None:
    context.guard_profile = _make_guarded_profile(
        max_tool_calls_per_step=limit,
    )


@given("a profile with no guards")
def step_profile_no_guards(context: Context) -> None:
    context.guard_profile = AutomationProfile(name="test/no-guards")


@given('a profile with denylist "{tools}"')
def step_profile_denylist(context: Context, tools: str) -> None:
    context.guard_profile = _make_guarded_profile(
        tool_denylist=tools.split(","),
    )


@given('a profile with allowlist "{tools}"')
def step_profile_allowlist(context: Context, tools: str) -> None:
    context.guard_profile = _make_guarded_profile(
        tool_allowlist=tools.split(","),
    )


@given("a profile with require_approval_for_writes true")
def step_profile_write_approval(context: Context) -> None:
    context.guard_profile = _make_guarded_profile(
        require_approval_for_writes=True,
    )


@given("a profile with require_approval_for_apply true")
def step_profile_apply_approval(context: Context) -> None:
    context.guard_profile = _make_guarded_profile(
        require_approval_for_apply=True,
    )


@given("a profile with max_total_cost {cost:g}")
def step_profile_max_cost(context: Context, cost: float) -> None:
    context.guard_profile = _make_guarded_profile(
        max_total_cost=cost,
    )


# -- Guard check steps --


@when('I check guard for tool "{tool}" with {calls:d} calls so far')
def step_check_guard_calls(
    context: Context,
    tool: str,
    calls: int,
) -> None:
    context.guard_result = context.guard_profile.check_guard(
        tool_name=tool,
        calls_so_far=calls,
    )


@when('I check guard for tool "{tool}" with {calls:d} calls so far in "{scope}" scope')
def step_check_guard_calls_with_scope(
    context: Context,
    tool: str,
    calls: int,
    scope: str,
) -> None:
    context.guard_result = context.guard_profile.check_guard(
        tool_name=tool,
        calls_so_far=calls,
        scope=GuardScope(scope),
    )


@when('I check guard for tool "{tool}"')
def step_check_guard_simple(context: Context, tool: str) -> None:
    context.guard_result = context.guard_profile.check_guard(
        tool_name=tool,
    )


@when('I check guard for tool "{tool}" with is_write {is_w}')
def step_check_guard_write(
    context: Context,
    tool: str,
    is_w: str,
) -> None:
    context.guard_result = context.guard_profile.check_guard(
        tool_name=tool,
        is_write=is_w.lower() == "true",
    )


@when('I check guard for tool "{tool}" with cost {cost:g}')
def step_check_guard_cost(
    context: Context,
    tool: str,
    cost: float,
) -> None:
    context.guard_result = context.guard_profile.check_guard(
        tool_name=tool,
        cost_so_far=cost,
    )


# -- Guard result assertions --


@then("the guard result should be allowed")
def step_guard_allowed(context: Context) -> None:
    assert context.guard_result.allowed is True, (
        f"Expected allowed, got: {context.guard_result}"
    )


@then("the guard result should not be allowed")
def step_guard_not_allowed(context: Context) -> None:
    assert context.guard_result.allowed is False, (
        f"Expected not allowed, got: {context.guard_result}"
    )


@then('the guard reason should mention "{text}"')
def step_guard_reason_mention(context: Context, text: str) -> None:
    reason = context.guard_result.reason or ""
    assert text in reason, f"Expected reason to mention '{text}', got: '{reason}'"


@then("the guard should require approval")
def step_guard_requires_approval(context: Context) -> None:
    assert context.guard_result.requires_approval is True


# -- Effective profile resolution steps --


@given('a profile service with profiles "{p1}" and "{p2}"')
def step_service_with_profiles(
    context: Context,
    p1: str,
    p2: str,
) -> None:
    context.profile_service = AutomationProfileService()


@given('a profile service with global default "{name}"')
def step_service_global_default(context: Context, name: str) -> None:
    context.profile_service = AutomationProfileService(
        global_default=name,
    )


@given("a profile service")
def step_service_default(context: Context) -> None:
    context.profile_service = AutomationProfileService()


@when('I resolve effective profile with plan "{name}"')
def step_resolve_plan(context: Context, name: str) -> None:
    context.resolved_profile = context.profile_service.get_effective_profile(
        plan_profile=name,
    )


@when('I resolve effective profile with action "{name}"')
def step_resolve_action(context: Context, name: str) -> None:
    context.resolved_profile = context.profile_service.get_effective_profile(
        action_profile=name,
    )


@when('I resolve effective profile with default "{name}"')
def step_resolve_default(context: Context, name: str) -> None:
    context.resolved_profile = context.profile_service.get_effective_profile(
        default_profile=name,
    )


@when("I resolve effective profile with no overrides")
def step_resolve_no_overrides(context: Context) -> None:
    context.resolved_profile = context.profile_service.get_effective_profile()


@then('the resolved guard profile name should be "{name}"')
def step_resolved_guard_name(context: Context, name: str) -> None:
    assert context.resolved_profile.name == name, (
        f"Expected '{name}', got '{context.resolved_profile.name}'"
    )


# -- YAML loading steps --


@given("a YAML profile file with guards")
def step_yaml_file_with_guards(context: Context) -> None:
    config = {
        "name": "test/yaml-guards",
        "description": "YAML test profile",
        "guards": {
            "max_tool_calls_per_step": 10,
            "max_total_cost": 50.0,
            "require_approval_for_writes": True,
        },
    }
    tmp_path = tempfile.mktemp(suffix=".yaml")
    with open(tmp_path, "w") as fh:
        yaml.dump(config, fh)
    context.yaml_path = tmp_path


@when("I load the profile from YAML")
def step_load_yaml(context: Context) -> None:
    context.loaded_profile = AutomationProfile.from_yaml(
        context.yaml_path,
    )


@when("I load the profile via service from_yaml")
def step_load_yaml_service(context: Context) -> None:
    context.loaded_profile = AutomationProfileService.from_yaml(
        context.yaml_path,
    )


@then("the loaded profile should have guards")
def step_loaded_has_guards(context: Context) -> None:
    assert context.loaded_profile.guards is not None


@then("the loaded profile guards max_tool_calls_per_step should be {val:d}")
def step_loaded_max_calls(context: Context, val: int) -> None:
    assert context.loaded_profile.guards is not None
    actual = context.loaded_profile.guards.max_tool_calls_per_step
    assert actual == val, f"Expected {val}, got {actual}"


# -- Built-in immutability steps --


@when('I try to update built-in profile "{name}"')
def step_try_update_builtin(context: Context, name: str) -> None:
    context.service_error = None
    try:
        context.profile_service.update_profile(name, {"name": name})
    except CAValidationError as e:
        context.service_error = e


@when('I try to delete built-in profile "{name}"')
def step_try_delete_builtin(context: Context, name: str) -> None:
    context.service_error = None
    try:
        context.profile_service.delete_profile(name)
    except CAValidationError as e:
        context.service_error = e


@then("a profile service validation error should be raised")
def step_service_validation_error(context: Context) -> None:
    assert context.service_error is not None, "Expected a ValidationError"


# -- Guard model validation steps --


@when("I try to create a guard with max_tool_calls {val:d}")
def step_try_guard_max_calls(context: Context, val: int) -> None:
    context.guard_error = None
    context.guard_model = None
    try:
        context.guard_model = AutomationGuard(
            max_tool_calls_per_step=val,
        )
    except ValidationError as e:
        context.guard_error = e


@when("I try to create a guard with max_total_cost {val:g}")
def step_try_guard_max_cost(context: Context, val: float) -> None:
    context.guard_error = None
    context.guard_model = None
    try:
        context.guard_model = AutomationGuard(max_total_cost=val)
    except ValidationError as e:
        context.guard_error = e


@when("I create a guard with max_tool_calls {val:d}")
def step_create_guard_max_calls(context: Context, val: int) -> None:
    context.guard_model = AutomationGuard(
        max_tool_calls_per_step=val,
    )


@when("I create a guard with defaults")
def step_create_guard_defaults(context: Context) -> None:
    context.guard_model = AutomationGuard()


@then("a guard model validation error should be raised")
def step_guard_model_validation_error(context: Context) -> None:
    assert context.guard_error is not None, "Expected a validation error"


@then("the guard result allowed should be true")
def step_result_allowed_true(context: Context) -> None:
    assert context.guard_result.allowed is True


@then("the guard result allowed should be false")
def step_result_allowed_false(context: Context) -> None:
    assert context.guard_result.allowed is False


@then("the guard result reason should be None")
def step_result_reason_none(context: Context) -> None:
    assert context.guard_result.reason is None


@then('the guard result reason should be "{expected}"')
def step_result_reason_value(context: Context, expected: str) -> None:
    assert context.guard_result.reason == expected


@then("the guard result requires_approval should be true")
def step_result_requires_approval(context: Context) -> None:
    assert context.guard_result.requires_approval is True


# -- Service evaluate_guard steps --


@when('I evaluate guard for profile "{name}" tool "{tool}"')
def step_evaluate_guard(context: Context, name: str, tool: str) -> None:
    context.guard_result = context.profile_service.evaluate_guard(
        profile_name=name,
        tool_name=tool,
    )


@when("I try to evaluate guard with empty profile name")
def step_evaluate_empty_profile(context: Context) -> None:
    context.service_error = None
    try:
        context.profile_service.evaluate_guard(
            profile_name="",
            tool_name="tool",
        )
    except CAValidationError as e:
        context.service_error = e


@when("I try to evaluate guard with empty tool name")
def step_evaluate_empty_tool(context: Context) -> None:
    context.service_error = None
    try:
        context.profile_service.evaluate_guard(
            profile_name="manual",
            tool_name="",
        )
    except CAValidationError as e:
        context.service_error = e


# -- Profile with guards from config --


@when("I create a profile from config with guards")
def step_profile_config_guards(context: Context) -> None:
    config: dict[str, Any] = {
        "name": "test/config-guards",
        "guards": {
            "tool_denylist": ["dangerous"],
            "require_approval_for_writes": True,
        },
    }
    context.config_profile = AutomationProfile.from_config(config)


@then("the profile should have guards configured")
def step_profile_has_guards(context: Context) -> None:
    assert context.config_profile.guards is not None


@then("the profile guards should have denylist")
def step_profile_guards_denylist(context: Context) -> None:
    assert context.config_profile.guards is not None
    assert context.config_profile.guards.tool_denylist is not None
    assert len(context.config_profile.guards.tool_denylist) > 0


# -- check_guard argument validation --


@when("I try to check guard with negative cost")
def step_check_guard_negative_cost(context: Context) -> None:
    context.guard_arg_error = None
    try:
        context.guard_profile.check_guard(
            tool_name="tool",
            cost_so_far=-1.0,
        )
    except ValueError as e:
        context.guard_arg_error = e


@when("I try to check guard with negative calls")
def step_check_guard_negative_calls(context: Context) -> None:
    context.guard_arg_error = None
    try:
        context.guard_profile.check_guard(
            tool_name="tool",
            calls_so_far=-1,
        )
    except ValueError as e:
        context.guard_arg_error = e


@then("a guard argument error should be raised")
def step_guard_arg_error(context: Context) -> None:
    assert context.guard_arg_error is not None, (
        "Expected a ValueError for invalid arguments"
    )


# -- Missing model construction / assertion steps --


@then("the guard should be created")
def step_guard_should_be_created(context: Context) -> None:
    assert context.guard_model is not None, "Guard was not created"


@then("the guard max_tool_calls_per_step should be None")
def step_guard_max_calls_none(context: Context) -> None:
    assert context.guard_model.max_tool_calls_per_step is None


@then("the guard max_total_cost should be None")
def step_guard_max_cost_none(context: Context) -> None:
    assert context.guard_model.max_total_cost is None


@when("I create a guard result allowed true")
def step_create_guard_result_allowed(context: Context) -> None:
    context.guard_result = GuardResult(allowed=True)


@when('I create a guard result denied with reason "{reason}"')
def step_create_guard_result_denied(context: Context, reason: str) -> None:
    context.guard_result = GuardResult(
        allowed=False,
        reason=reason,
        requires_approval=True,
    )