Files
cleveragents-core/features/steps/tool_env_preferences_steps.py
T
brent.edwards 8d108cb5d1
CI / lint (push) Successful in 21s
CI / build (push) Successful in 27s
CI / quality (push) Successful in 30s
CI / typecheck (push) Successful in 40s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 54s
CI / integration_tests (push) Successful in 3m57s
CI / e2e_tests (push) Successful in 4m51s
CI / unit_tests (push) Successful in 5m23s
CI / docker (push) Successful in 1m2s
CI / coverage (push) Successful in 6m57s
CI / benchmark-publish (push) Successful in 19m29s
feat(tool): add tool-level execution environment preferences (#970)
## Summary

Add tool-level execution environment preferences with four modes: `required`, `preferred`, `specific`, and `none`.

### Changes

**New model** (`domain/models/core/execution_environment_preference.py`):
- `EnvironmentPreferenceMode` enum: REQUIRED, PREFERRED, SPECIFIC, NONE
- `ExecutionEnvironmentPreference` frozen Pydantic model with `mode` and `target_resource` fields
- Model validator ensures `target_resource` required iff mode is SPECIFIC

**ToolSpec & Tool model updates:**
- Added `execution_environment: ExecutionEnvironmentPreference` field to both `ToolSpec` (runtime) and `Tool` (domain)
- `Tool.from_config()` parses `execution_environment` from YAML config dicts

**ToolRunner integration** (`tool/runner.py`):
- Before calling `_env_resolver.resolve()`, checks `spec.execution_environment.mode`:
  - REQUIRED: raises `ContainerUnavailableError` if resolved env is not CONTAINER
  - PREFERRED: tries container, gracefully falls back to host
  - SPECIFIC: overrides `tool_env` with `target_resource`
  - NONE: current behavior (caller-supplied tool_env)

### Tests

- **27 Behave scenarios** covering model validation, serialization, preference routing, YAML parsing, error cases
- **12 Robot integration tests** covering CLI tool preference display and end-to-end routing

### Quality Gates

| Session | Result |
|---|---|
| `nox -s lint` | PASS |
| `nox -s typecheck` | PASS (0 errors) |
| `nox -s unit_tests` | PASS (10,833 scenarios) |
| `nox -s integration_tests` | PASS (12 new tests) |

Closes #879

Reviewed-on: #970
Co-authored-by: Brent Edwards <brent.edwards@cleverthis.com>
Co-committed-by: Brent Edwards <brent.edwards@cleverthis.com>
2026-03-20 23:51:46 +00:00

477 lines
16 KiB
Python

"""Step definitions for tool-level execution environment preference tests."""
from __future__ import annotations
from enum import StrEnum
from typing import Any
from unittest.mock import create_autospec
from behave import given, then, when
from behave.runner import Context
# ---------------------------------------------------------------------------
# EnvironmentPreferenceMode enum steps
# ---------------------------------------------------------------------------
@given("I import EnvironmentPreferenceMode")
def step_import_pref_mode(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
)
context.pref_mode_cls = EnvironmentPreferenceMode
@then('the NONE preference value should be "{value}"')
def step_none_pref_value(context: Context, value: str) -> None:
assert context.pref_mode_cls.NONE.value == value
@then('the REQUIRED preference value should be "{value}"')
def step_required_pref_value(context: Context, value: str) -> None:
assert context.pref_mode_cls.REQUIRED.value == value
@then('the PREFERRED preference value should be "{value}"')
def step_preferred_pref_value(context: Context, value: str) -> None:
assert context.pref_mode_cls.PREFERRED.value == value
@then('the SPECIFIC preference value should be "{value}"')
def step_specific_pref_value(context: Context, value: str) -> None:
assert context.pref_mode_cls.SPECIFIC.value == value
@then("EnvironmentPreferenceMode should be a StrEnum subclass")
def step_pref_mode_is_strenum(context: Context) -> None:
assert issubclass(context.pref_mode_cls, StrEnum)
# ---------------------------------------------------------------------------
# ExecutionEnvironmentPreference creation steps
# ---------------------------------------------------------------------------
@given("I create a default ExecutionEnvironmentPreference")
def step_create_default_pref(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
ExecutionEnvironmentPreference,
)
context.pref = ExecutionEnvironmentPreference()
@given('I create an ExecutionEnvironmentPreference with mode "{mode}"')
def step_create_pref_with_mode(context: Context, mode: str) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
context.pref = ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode(mode),
)
@given('I create a specific ExecutionEnvironmentPreference targeting "{target}"')
def step_create_specific_pref(context: Context, target: str) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
context.pref = ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.SPECIFIC,
target_resource=target,
)
@when(
'I try to create an ExecutionEnvironmentPreference with mode "{mode}"'
' and target "{target}"'
)
def step_try_create_pref_with_target(context: Context, mode: str, target: str) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
try:
ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode(mode),
target_resource=target,
)
context.pref_error = None
except ValueError as exc:
context.pref_error = str(exc)
@when(
'I try to create an ExecutionEnvironmentPreference with mode "specific"'
" and no target"
)
def step_try_create_specific_no_target(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
try:
ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.SPECIFIC,
)
context.pref_error = None
except ValueError as exc:
context.pref_error = str(exc)
@then('the preference creation should fail with "{message}"')
def step_pref_creation_failed(context: Context, message: str) -> None:
assert context.pref_error is not None, "Expected creation to fail"
assert message in context.pref_error, (
f"Expected '{message}' in error, got: {context.pref_error}"
)
@then('the preference mode should be "{mode}"')
def step_pref_mode_is(context: Context, mode: str) -> None:
assert context.pref.mode.value == mode
@then("the preference target_resource should be None")
def step_pref_target_none(context: Context) -> None:
assert context.pref.target_resource is None
@then('the preference target_resource should be "{target}"')
def step_pref_target_is(context: Context, target: str) -> None:
assert context.pref.target_resource == target
# ---------------------------------------------------------------------------
# Frozen model steps
# ---------------------------------------------------------------------------
@when("I try to mutate the preference mode")
def step_try_mutate_pref(context: Context) -> None:
from pydantic import ValidationError
try:
context.pref.mode = "required" # type: ignore[misc]
context.mutation_failed = False
except (ValidationError, TypeError):
context.mutation_failed = True
@then("the mutation should fail")
def step_mutation_failed(context: Context) -> None:
assert context.mutation_failed is True
# ---------------------------------------------------------------------------
# ToolSpec field steps
# ---------------------------------------------------------------------------
def _make_handler(inputs: dict[str, Any]) -> dict[str, Any]:
return {"result": "ok"}
@given("I create a ToolSpec with default execution_environment")
def step_toolspec_default_env(context: Context) -> None:
from cleveragents.tool.runtime import ToolSpec
context.tool_spec = ToolSpec(
name="test/tool",
description="A test tool",
handler=_make_handler,
)
@given("I create a ToolSpec with required execution_environment")
def step_toolspec_required_env(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
from cleveragents.tool.runtime import ToolSpec
context.tool_spec = ToolSpec(
name="test/tool",
description="A test tool",
handler=_make_handler,
execution_environment=ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.REQUIRED,
),
)
@then('the ToolSpec execution_environment mode should be "{mode}"')
def step_toolspec_env_mode(context: Context, mode: str) -> None:
assert context.tool_spec.execution_environment.mode.value == mode
# ---------------------------------------------------------------------------
# Domain Tool field steps
# ---------------------------------------------------------------------------
@given("I create a domain Tool with default execution_environment")
def step_domain_tool_default_env(context: Context) -> None:
from cleveragents.domain.models.core.tool import Tool, ToolSource
context.domain_tool = Tool(
name="local/test-tool",
description="A test tool",
source=ToolSource.BUILTIN,
)
@given("I create a domain Tool with required execution_environment")
def step_domain_tool_required_env(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
from cleveragents.domain.models.core.tool import Tool, ToolSource
context.domain_tool = Tool(
name="local/test-tool",
description="A test tool",
source=ToolSource.BUILTIN,
execution_environment=ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.REQUIRED,
),
)
@then('the domain Tool execution_environment mode should be "{mode}"')
def step_domain_tool_env_mode(context: Context, mode: str) -> None:
assert context.domain_tool.execution_environment.mode.value == mode
# ---------------------------------------------------------------------------
# Tool.from_config steps
# ---------------------------------------------------------------------------
@given('I have a tool config dict with execution_environment mode "{mode}"')
def step_config_with_env_mode(context: Context, mode: str) -> None:
context.tool_config = {
"name": "local/test-tool",
"description": "A test tool",
"source": "builtin",
"execution_environment": {"mode": mode},
}
@given('I have a tool config dict with execution_environment targeting "{target}"')
def step_config_with_env_mode_target(context: Context, target: str) -> None:
context.tool_config = {
"name": "local/test-tool",
"description": "A test tool",
"source": "builtin",
"execution_environment": {"mode": "specific", "target_resource": target},
}
@given("I have a tool config dict without execution_environment")
def step_config_without_env(context: Context) -> None:
context.tool_config = {
"name": "local/test-tool",
"description": "A test tool",
"source": "builtin",
}
@when("I create a Tool from the config")
def step_tool_from_config(context: Context) -> None:
from cleveragents.domain.models.core.tool import Tool
context.domain_tool = Tool.from_config(context.tool_config)
@then('the tool execution_environment mode should be "{mode}"')
def step_tool_env_mode(context: Context, mode: str) -> None:
assert context.domain_tool.execution_environment.mode.value == mode
@then('the tool execution_environment target_resource should be "{target}"')
def step_tool_env_target(context: Context, target: str) -> None:
assert context.domain_tool.execution_environment.target_resource == target
# ---------------------------------------------------------------------------
# as_cli_dict steps
# ---------------------------------------------------------------------------
@when("I call as_cli_dict on the tool")
def step_call_cli_dict(context: Context) -> None:
context.cli_dict = context.domain_tool.as_cli_dict()
@then('the cli dict should not contain "{key}"')
def step_cli_dict_no_key(context: Context, key: str) -> None:
assert key not in context.cli_dict
@then('the cli dict should contain execution_environment with mode "{mode}"')
def step_cli_dict_env_mode(context: Context, mode: str) -> None:
assert "execution_environment" in context.cli_dict
assert context.cli_dict["execution_environment"]["mode"] == mode
# ---------------------------------------------------------------------------
# ToolRunner preference routing steps
# ---------------------------------------------------------------------------
@given("I have a ToolRunner with a tool with NONE preference")
def step_runner_none_pref(context: Context) -> None:
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
spec = ToolSpec(
name="test/none-pref",
description="Tool with no env preference",
handler=_make_handler,
)
registry.register(spec)
context.runner = ToolRunner(registry=registry)
context.runner.activate("test/none-pref")
context.tool_name = "test/none-pref"
@when("I execute the tool with no caller env")
def step_execute_no_caller_env(context: Context) -> None:
context.result = context.runner.execute(context.tool_name, {"input": "value"})
@then("the tool preference routing should succeed on host")
def step_tool_success_host(context: Context) -> None:
assert context.result.success is True
@given("I have a ToolRunner with a tool with REQUIRED preference and no container")
def step_runner_required_no_container(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
spec = ToolSpec(
name="test/required-pref",
description="Tool requiring container",
handler=_make_handler,
execution_environment=ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.REQUIRED,
),
)
registry.register(spec)
context.runner = ToolRunner(registry=registry)
context.runner.activate("test/required-pref")
context.tool_name = "test/required-pref"
@when("I execute the required-preference tool")
def step_execute_required(context: Context) -> None:
context.result = context.runner.execute(context.tool_name, {"input": "value"})
@then("the execution should fail with container required error")
def step_fail_container_required(context: Context) -> None:
assert context.result.success is False
assert "requires container execution" in (context.result.error or "")
@given("I have a ToolRunner with a tool with PREFERRED preference and no container")
def step_runner_preferred_no_container(context: Context) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
spec = ToolSpec(
name="test/preferred-pref",
description="Tool preferring container",
handler=_make_handler,
execution_environment=ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.PREFERRED,
),
)
registry.register(spec)
context.runner = ToolRunner(registry=registry)
context.runner.activate("test/preferred-pref")
context.tool_name = "test/preferred-pref"
@when("I execute the preferred-preference tool")
def step_execute_preferred(context: Context) -> None:
context.result = context.runner.execute(context.tool_name, {"input": "value"})
@given('I have a ToolRunner with a tool with SPECIFIC preference targeting "{target}"')
def step_runner_specific_pref(context: Context, target: str) -> None:
from cleveragents.domain.models.core.execution_environment_preference import (
EnvironmentPreferenceMode,
ExecutionEnvironmentPreference,
)
from cleveragents.tool.registry import ToolRegistry
from cleveragents.tool.runner import ToolRunner
from cleveragents.tool.runtime import ToolSpec
registry = ToolRegistry()
spec = ToolSpec(
name="test/specific-pref",
description="Tool targeting specific container",
handler=_make_handler,
execution_environment=ExecutionEnvironmentPreference(
mode=EnvironmentPreferenceMode.SPECIFIC,
target_resource=target,
),
)
registry.register(spec)
# Provide a mock container executor so container routing works
from cleveragents.tool.container_executor import ContainerToolExecutor
from cleveragents.tool.runtime import ToolResult
mock_executor = create_autospec(ContainerToolExecutor, instance=True)
mock_executor.execute_tool.return_value = ToolResult(
success=True,
output={"result": "container-ok"},
duration_ms=1.0,
)
context.runner = ToolRunner(registry=registry, container_executor=mock_executor)
context.runner.activate("test/specific-pref")
context.tool_name = "test/specific-pref"
@when("I execute the specific-preference tool with container available")
def step_execute_specific_container(context: Context) -> None:
context.result = context.runner.execute(
context.tool_name,
{"input": "value"},
linked_resource_types=["container-instance"],
project_name="test-project",
)
@then("the tool preference routing should succeed in container")
def step_tool_container_mode(context: Context) -> None:
assert context.result.success is True