Files
cleveragents-core/features/steps/tool_lifecycle_runtime_steps.py
CoreRasurae 1e987a2009
CI / build (push) Successful in 19s
CI / helm (push) Successful in 22s
CI / security (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / e2e_tests (push) Has been cancelled
CI / quality (push) Has been cancelled
CI / typecheck (push) Has been cancelled
CI / integration_tests (push) Has been cancelled
CI / unit_tests (push) Has been cancelled
CI / benchmark-publish (push) Has been cancelled
CI / benchmark-regression (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
CI / coverage (push) Has been cancelled
feat(events): wire all 38 domain event emissions into services (#1215)
Co-authored-by: Luis Mendes <luis.mendes@cleverthis.com>
Co-committed-by: Luis Mendes <luis.mendes@cleverthis.com>
2026-04-02 16:53:03 +00:00

1046 lines
37 KiB
Python

"""Step definitions for tool lifecycle runtime feature tests."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from behave import given, then, when
from behave.runner import Context
from cleveragents.domain.models.core.tool import (
Tool,
ToolCapability,
ToolSource,
ToolType,
)
from cleveragents.infrastructure.events.reactive import ReactiveEventBus
from cleveragents.tool.context import (
BoundResource,
CancellationToken,
Change,
ChangeOperation,
ToolCancelledError,
ToolExecutionContext,
ToolExecutionTrace,
)
from cleveragents.tool.lifecycle import (
ToolAccessDeniedError,
ToolActivationError,
ToolCheckpointRequiredError,
ToolDescriptor,
ToolExecutionError,
ToolLifecycleCache,
ToolResult,
ToolRuntime,
ToolRuntimeError,
)
from cleveragents.tool.schema_validator import (
ToolSchemaValidationError,
validate_tool_input,
validate_tool_output,
)
# ---------------------------------------------------------------------------
# Mock tool instance
# ---------------------------------------------------------------------------
class MockToolInstance:
"""A mock ToolInstance for testing."""
def __init__(self, name: str = "mock/tool", read_only: bool = True) -> None:
self.name = name
self.read_only = read_only
self.activated = False
self.deactivated = False
self.activate_count = 0
self.execute_count = 0
self.produce_changes = False
self.return_invalid_output = False
def discover(self) -> ToolDescriptor:
return ToolDescriptor(
name=self.name,
description=f"Mock tool {self.name}",
capability=ToolCapability(read_only=self.read_only),
)
def activate(self, ctx: ToolExecutionContext) -> None:
self.activated = True
self.activate_count += 1
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
self.execute_count += 1
changes: list[Change] = []
if self.produce_changes:
changes.append(
Change(
operation=ChangeOperation.MODIFY,
resource_id="res-001",
path=params.get("path", "unknown"),
)
)
data: Any = f"result for {params.get('path', 'unknown')}"
if self.return_invalid_output:
data = 42 # Intentionally wrong type for schema validation
return ToolResult(success=True, data=data, changes=changes)
def deactivate(self, ctx: ToolExecutionContext) -> None:
self.deactivated = True
class FailingActivateMock(MockToolInstance):
"""Mock that raises on activate()."""
def activate(self, ctx: ToolExecutionContext) -> None:
raise RuntimeError("activate boom")
class FailingExecuteMock(MockToolInstance):
"""Mock that raises on execute()."""
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
raise RuntimeError("execute boom")
class FailingDeactivateMock(MockToolInstance):
"""Mock that raises on deactivate()."""
def deactivate(self, ctx: ToolExecutionContext) -> None:
raise RuntimeError("deactivate boom")
class ErrorResultMock(MockToolInstance):
"""Mock that returns a ToolResult with error set."""
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
return ToolResult(success=False, data=None, error="something went wrong")
# ---------------------------------------------------------------------------
# ToolExecutionContext steps
# ---------------------------------------------------------------------------
@given('I create a tool execution context with plan_id "{plan_id}"')
def step_create_context(context: Context, plan_id: str) -> None:
context.tool_ctx = ToolExecutionContext(plan_id=plan_id)
@given('I create a tool execution context with plan_id "{plan_id}" and read_only True')
def step_create_context_read_only(context: Context, plan_id: str) -> None:
context.tool_ctx = ToolExecutionContext(plan_id=plan_id, plan_read_only=True)
@given(
'I create a tool execution context with plan_id "{plan_id}" and require_checkpoints True'
)
def step_create_context_checkpoints(context: Context, plan_id: str) -> None:
context.tool_ctx = ToolExecutionContext(plan_id=plan_id, require_checkpoints=True)
@then('the context plan_id should be "{expected}"')
def step_check_context_plan_id(context: Context, expected: str) -> None:
assert context.tool_ctx.plan_id == expected
@then("the context should not be read-only")
def step_check_context_not_read_only(context: Context) -> None:
assert not context.tool_ctx.plan_read_only
@then("the context should be read-only")
def step_check_context_read_only(context: Context) -> None:
assert context.tool_ctx.plan_read_only
@then("the context should not require checkpoints")
def step_check_context_no_checkpoints(context: Context) -> None:
assert not context.tool_ctx.require_checkpoints
@then("the context should require checkpoints")
def step_check_context_checkpoints(context: Context) -> None:
assert context.tool_ctx.require_checkpoints
@then("the context should have {count:d} changes")
def step_check_context_change_count(context: Context, count: int) -> None:
assert len(context.tool_ctx.changes) == count
@then("the context should have {count:d} traces")
def step_check_context_trace_count(context: Context, count: int) -> None:
assert len(context.tool_ctx.traces) == count
@then("the context cancellation token should not be cancelled")
def step_check_token_not_cancelled(context: Context) -> None:
assert not context.tool_ctx.cancellation_token.is_cancelled
# ── Change recording ──────────────────────────────────────────────────
@when('I record a change with operation "{op}" and resource_id "{res_id}"')
def step_record_change(context: Context, op: str, res_id: str) -> None:
change = Change(operation=ChangeOperation(op), resource_id=res_id)
context.tool_ctx.record_change(change)
# ── Trace recording ──────────────────────────────────────────────────
@when('I add a trace for tool "{tool_name}" with success {success}')
def step_add_trace(context: Context, tool_name: str, success: str) -> None:
trace = ToolExecutionTrace(
tool_name=tool_name,
started_at=datetime.now(UTC).isoformat(),
success=success == "True",
)
context.tool_ctx.add_trace(trace)
# ── Resource binding ─────────────────────────────────────────────────
@given('I bind resource "{res_id}" to slot "{slot}" with type "{res_type}"')
def step_bind_resource(context: Context, res_id: str, slot: str, res_type: str) -> None:
bound = BoundResource(
slot_name=slot,
resource_id=res_id,
resource_type=res_type,
)
context.tool_ctx.resources[slot] = bound
@when('I get the resource for slot "{slot}"')
def step_get_resource(context: Context, slot: str) -> None:
context.bound_resource = context.tool_ctx.get_resource(slot)
@then('the bound resource_id should be "{expected}"')
def step_check_bound_resource_id(context: Context, expected: str) -> None:
assert context.bound_resource.resource_id == expected
@when('I try to get the resource for slot "{slot}"')
def step_try_get_resource(context: Context, slot: str) -> None:
try:
context.tool_ctx.get_resource(slot)
context.resource_error = None
except KeyError as exc:
context.resource_error = exc
@then('a KeyError should be raised with message containing "{text}"')
def step_check_key_error(context: Context, text: str) -> None:
assert context.resource_error is not None
assert text in str(context.resource_error)
# ── Context summary ──────────────────────────────────────────────────
@then("the context summary should show resource_count {count:d}")
def step_check_summary_resources(context: Context, count: int) -> None:
summary = context.tool_ctx.as_summary()
assert summary["resource_count"] == count
@then("the context summary should show change_count {count:d}")
def step_check_summary_changes(context: Context, count: int) -> None:
summary = context.tool_ctx.as_summary()
assert summary["change_count"] == count
@then("the context summary should show trace_count {count:d}")
def step_check_summary_traces(context: Context, count: int) -> None:
summary = context.tool_ctx.as_summary()
assert summary["trace_count"] == count
# ---------------------------------------------------------------------------
# BoundResource steps
# ---------------------------------------------------------------------------
@given(
'I create a bound resource with slot "{slot}" and resource_id "{res_id}" and type "{res_type}"'
)
def step_create_bound_resource(
context: Context, slot: str, res_id: str, res_type: str
) -> None:
context.bound_resource = BoundResource(
slot_name=slot,
resource_id=res_id,
resource_type=res_type,
)
@then('the bound resource slot_name should be "{expected}"')
def step_check_bound_slot(context: Context, expected: str) -> None:
assert context.bound_resource.slot_name == expected
@then('the bound resource access should be "{expected}"')
def step_check_bound_access(context: Context, expected: str) -> None:
assert context.bound_resource.access == expected
# ---------------------------------------------------------------------------
# CancellationToken steps
# ---------------------------------------------------------------------------
@given("I create a cancellation token")
def step_create_cancellation_token(context: Context) -> None:
context.cancel_token = CancellationToken()
@when("I cancel the token")
def step_cancel_token(context: Context) -> None:
context.cancel_token.cancel()
@then("the cancellation token should not be cancelled")
def step_check_token_not_cancelled_standalone(context: Context) -> None:
assert not context.cancel_token.is_cancelled
@then("the cancellation token should be cancelled")
def step_check_token_cancelled(context: Context) -> None:
assert context.cancel_token.is_cancelled
@then("calling check on the token should raise ToolCancelledError")
def step_check_token_raises(context: Context) -> None:
try:
context.cancel_token.check()
raise AssertionError("Expected ToolCancelledError")
except ToolCancelledError:
pass
# ---------------------------------------------------------------------------
# Change steps
# ---------------------------------------------------------------------------
@given('I create a change with operation "{op}" and resource_id "{res_id}"')
def step_create_change(context: Context, op: str, res_id: str) -> None:
context.change = Change(operation=ChangeOperation(op), resource_id=res_id)
@then('the change operation should be "{expected}"')
def step_check_change_op(context: Context, expected: str) -> None:
assert context.change.operation == expected
@then('the change resource_id should be "{expected}"')
def step_check_change_res_id(context: Context, expected: str) -> None:
assert context.change.resource_id == expected
@then("the change timestamp should be a valid ISO-8601 string")
def step_check_change_timestamp(context: Context) -> None:
ts = context.change.timestamp
assert ts is not None
# Should parse without error
datetime.fromisoformat(ts)
# ---------------------------------------------------------------------------
# ToolDescriptor steps
# ---------------------------------------------------------------------------
@given('I create a tool descriptor with name "{name}" and description "{desc}"')
def step_create_descriptor(context: Context, name: str, desc: str) -> None:
context.descriptor = ToolDescriptor(name=name, description=desc)
@then('the descriptor name should be "{expected}"')
def step_check_descriptor_name(context: Context, expected: str) -> None:
assert context.descriptor.name == expected
@then('the descriptor source should be "{expected}"')
def step_check_descriptor_source(context: Context, expected: str) -> None:
assert context.descriptor.source == expected
# ---------------------------------------------------------------------------
# ToolResult steps
# ---------------------------------------------------------------------------
@given('I create a tool result with success {success} and data "{data}"')
def step_create_result_success(context: Context, success: str, data: str) -> None:
context.tool_result = ToolResult(success=success == "True", data=data)
@given('I create a tool result with success {success} and error "{error}"')
def step_create_result_error(context: Context, success: str, error: str) -> None:
context.tool_result = ToolResult(success=success == "True", error=error)
@then("the tool result should indicate success")
def step_check_result_success(context: Context) -> None:
assert context.tool_result.success
@then("the tool result should indicate failure")
def step_check_result_not_success(context: Context) -> None:
assert not context.tool_result.success
@then('the tool result data should be "{expected}"')
def step_check_result_data(context: Context, expected: str) -> None:
assert context.tool_result.data == expected
@then('the tool result error should be "{expected}"')
def step_check_result_error(context: Context, expected: str) -> None:
assert context.tool_result.error == expected
# ---------------------------------------------------------------------------
# JSON Schema validation steps
# ---------------------------------------------------------------------------
@given('I have a JSON schema requiring property "{prop}" of type "{prop_type}"')
def step_create_schema(context: Context, prop: str, prop_type: str) -> None:
context.json_schema = {
"type": "object",
"properties": {prop: {"type": prop_type}},
"required": [prop],
}
@when('I validate input with path "{value}" against the schema')
def step_validate_input_string(context: Context, value: str) -> None:
try:
validate_tool_input({"path": value}, context.json_schema)
context.schema_error = None
except ToolSchemaValidationError as exc:
context.schema_error = exc
@when("I validate input with path {value:d} against the schema")
def step_validate_input_int(context: Context, value: int) -> None:
try:
validate_tool_input({"path": value}, context.json_schema)
context.schema_error = None
except ToolSchemaValidationError as exc:
context.schema_error = exc
@then("the input validation should succeed")
def step_check_input_valid(context: Context) -> None:
assert context.schema_error is None
@then("the input validation should fail with ToolSchemaValidationError")
def step_check_input_invalid(context: Context) -> None:
assert context.schema_error is not None
assert isinstance(context.schema_error, ToolSchemaValidationError)
@when('I validate output with content "{value}" against the schema')
def step_validate_output_string(context: Context, value: str) -> None:
try:
validate_tool_output({"content": value}, context.json_schema)
context.schema_error = None
except ToolSchemaValidationError as exc:
context.schema_error = exc
@when("I validate output with content {value:d} against the schema")
def step_validate_output_int(context: Context, value: int) -> None:
try:
validate_tool_output({"content": value}, context.json_schema)
context.schema_error = None
except ToolSchemaValidationError as exc:
context.schema_error = exc
@then("the output validation should succeed")
def step_check_output_valid(context: Context) -> None:
assert context.schema_error is None
@then("the output validation should fail with ToolSchemaValidationError")
def step_check_output_invalid(context: Context) -> None:
assert context.schema_error is not None
assert isinstance(context.schema_error, ToolSchemaValidationError)
@then("the schema validation error should have errors list")
def step_check_schema_error_details(context: Context) -> None:
assert context.schema_error is not None
assert len(context.schema_error.errors) > 0
@then("the schema validation error should reference the schema")
def step_check_schema_error_schema(context: Context) -> None:
assert context.schema_error is not None
assert context.schema_error.schema is not None
# ---------------------------------------------------------------------------
# ToolLifecycleCache steps
# ---------------------------------------------------------------------------
@given("I create a tool lifecycle cache")
def step_create_cache(context: Context) -> None:
context.lifecycle_cache = ToolLifecycleCache()
@given('I have a mock tool instance named "{name}"')
def step_create_mock_instance(context: Context, name: str) -> None:
context.mock_instance = MockToolInstance(name=name)
@when('I put the instance in cache for plan "{plan_id}" and tool "{tool_name}"')
def step_cache_put(context: Context, plan_id: str, tool_name: str) -> None:
context.lifecycle_cache.put(plan_id, tool_name, context.mock_instance)
@when('I put the second instance in cache for plan "{plan_id}" and tool "{tool_name}"')
def step_cache_put_second(context: Context, plan_id: str, tool_name: str) -> None:
# Use a separate instance if available, otherwise create one
inst = getattr(context, "mock_instance_2", None)
if inst is None:
inst = MockToolInstance(name=tool_name)
context.mock_instance_2 = inst
context.lifecycle_cache.put(plan_id, tool_name, inst)
@then("the cache should have {count:d} plans")
def step_check_cache_plan_count(context: Context, count: int) -> None:
assert context.lifecycle_cache.plan_count == count
@then(
'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return it'
)
def step_cache_get_returns(context: Context, plan_id: str, tool_name: str) -> None:
result = context.lifecycle_cache.get(plan_id, tool_name)
assert result is not None
@then(
'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return None'
)
def step_cache_get_returns_none(context: Context, plan_id: str, tool_name: str) -> None:
result = context.lifecycle_cache.get(plan_id, tool_name)
assert result is None
@when('I increment execution for plan "{plan_id}" and tool "{tool_name}"')
def step_cache_increment(context: Context, plan_id: str, tool_name: str) -> None:
context.lifecycle_cache.increment_execution(plan_id, tool_name)
@then(
'the cache stats for plan "{plan_id}" should show tool "{tool_name}" with execution_count {count:d}'
)
def step_check_cache_stats_exec(
context: Context, plan_id: str, tool_name: str, count: int
) -> None:
stats = context.lifecycle_cache.get_stats(plan_id)
assert stats["tools"][tool_name]["execution_count"] == count
@then('the plan tools for plan "{plan_id}" should include "{tool_name}"')
def step_check_plan_tools_include(
context: Context, plan_id: str, tool_name: str
) -> None:
tools = context.lifecycle_cache.get_plan_tools(plan_id)
assert tool_name in tools
@when('I remove tool "{tool_name}" from plan "{plan_id}"')
def step_cache_remove_tool(context: Context, tool_name: str, plan_id: str) -> None:
context.lifecycle_cache.remove(plan_id, tool_name)
@when('I remove plan "{plan_id}" from cache')
def step_cache_remove_plan(context: Context, plan_id: str) -> None:
context.lifecycle_cache.remove_plan(plan_id)
# ---------------------------------------------------------------------------
# ToolRuntime steps
# ---------------------------------------------------------------------------
def _make_tool(
name: str,
*,
read_only: bool = True,
writes: bool = False,
checkpointable: bool = False,
input_schema: dict[str, Any] | None = None,
output_schema: dict[str, Any] | None = None,
) -> Tool:
"""Create a minimal Tool domain model for tests."""
return Tool(
name=name,
description=f"Test tool {name}",
source=ToolSource.BUILTIN,
tool_type=ToolType.TOOL,
capability=ToolCapability(
read_only=read_only,
writes=writes,
checkpointable=checkpointable,
),
input_schema=input_schema,
output_schema=output_schema,
)
@given("I create a tool runtime")
def step_create_runtime(context: Context) -> None:
context.tool_runtime = ToolRuntime(event_bus=ReactiveEventBus())
context.mock_instances = dict[str, MockToolInstance]()
@given('I register a builtin tool "{name}" with a mock instance')
def step_register_tool(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=False)
mock = MockToolInstance(name=name, read_only=False)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given('I register a builtin tool "{name}" with a mock instance that is read_only')
def step_register_read_only_tool(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = MockToolInstance(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given('I register a builtin tool "{name}" that writes')
def step_register_writes_tool(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=False, writes=True)
mock = MockToolInstance(name=name, read_only=False)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given('I register a builtin tool "{name}" that produces changes')
def step_register_change_tool(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=False, writes=True)
mock = MockToolInstance(name=name, read_only=False)
mock.produce_changes = True
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given('I register a builtin tool with input schema "{name}"')
def step_register_tool_with_input_schema(context: Context, name: str) -> None:
schema = {
"type": "object",
"properties": {"path": {"type": "string"}},
"required": ["path"],
}
tool = _make_tool(name, read_only=True, input_schema=schema)
mock = MockToolInstance(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given('I register a builtin tool with output schema "{name}"')
def step_register_tool_with_output_schema(context: Context, name: str) -> None:
schema = {
"type": "object",
"properties": {"result": {"type": "string"}},
"required": ["result"],
}
tool = _make_tool(name, read_only=True, output_schema=schema)
mock = MockToolInstance(name=name, read_only=True)
mock.return_invalid_output = True # Will return int instead of dict
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@given(
'I register a second builtin tool "{name}" with a mock instance that is read_only'
)
def step_register_second_tool(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = MockToolInstance(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@when('I unregister tool "{name}"')
def step_unregister_tool(context: Context, name: str) -> None:
context.tool_runtime.unregister_tool(name)
@then('the tool list should contain "{name}"')
def step_check_tool_in_list(context: Context, name: str) -> None:
assert name in context.tool_runtime.list_tools()
@then('the tool list should not contain "{name}"')
def step_check_tool_not_in_list(context: Context, name: str) -> None:
assert name not in context.tool_runtime.list_tools()
@then('getting tool "{name}" should return the tool model')
def step_check_get_tool(context: Context, name: str) -> None:
tool = context.tool_runtime.get_tool(name)
assert tool is not None
assert tool.name == name
@then('getting tool "{name}" should return None')
def step_check_get_tool_none(context: Context, name: str) -> None:
assert context.tool_runtime.get_tool(name) is None
@when('I discover tool "{name}"')
def step_discover_tool(context: Context, name: str) -> None:
context.discovered_descriptor = context.tool_runtime.discover(name)
@then('the discovered descriptor name should be "{expected}"')
def step_check_discovered_name(context: Context, expected: str) -> None:
assert context.discovered_descriptor.name == expected
@when('I activate tool "{name}" in the runtime')
def step_activate_tool(context: Context, name: str) -> None:
context.tool_runtime.activate(name, context.tool_ctx)
@when('I activate tool "{name}" in the runtime again')
def step_activate_tool_again(context: Context, name: str) -> None:
context.tool_runtime.activate(name, context.tool_ctx)
@then("the mock instance should have been activated")
def step_check_mock_activated(context: Context) -> None:
# Find the first mock instance that was activated
for mock in context.mock_instances.values():
if mock.activated:
return
raise AssertionError("No mock instance was activated")
@then("the mock instance activate count should be {count:d}")
def step_check_mock_activate_count(context: Context, count: int) -> None:
total = sum(m.activate_count for m in context.mock_instances.values())
assert total == count
@when('I execute tool "{name}" with params path "{path}"')
def step_execute_tool(context: Context, name: str, path: str) -> None:
context.exec_result = context.tool_runtime.execute(
name, {"path": path}, context.tool_ctx
)
@when('I execute tool "{name}" with invalid params')
def step_execute_tool_invalid(context: Context, name: str) -> None:
# Pass integer instead of string for 'path'
context.exec_result = context.tool_runtime.execute(
name, {"path": 42}, context.tool_ctx
)
@then("the execution result should be successful")
def step_check_exec_success(context: Context) -> None:
assert context.exec_result.success
@then("the execution result should not be successful")
def step_check_exec_not_success(context: Context) -> None:
assert not context.exec_result.success
@then('the execution result error should contain "{text}"')
def step_check_exec_error_contains(context: Context, text: str) -> None:
assert context.exec_result.error is not None
assert text in context.exec_result.error
# ── Access denied / checkpoint required ──────────────────────────────
@when('I try to execute tool "{name}"')
def step_try_execute_tool(context: Context, name: str) -> None:
try:
context.tool_runtime.execute(name, {}, context.tool_ctx)
context.access_error = None
except ToolAccessDeniedError as exc:
context.access_error = exc
except ToolCheckpointRequiredError as exc:
context.access_error = exc
@then("a ToolAccessDeniedError should be raised")
def step_check_access_denied(context: Context) -> None:
assert isinstance(context.access_error, ToolAccessDeniedError)
@when('I try to activate tool "{name}" for checkpoint plan')
def step_try_activate_checkpoint(context: Context, name: str) -> None:
try:
context.tool_runtime.activate(name, context.tool_ctx)
context.checkpoint_error = None
except ToolCheckpointRequiredError as exc:
context.checkpoint_error = exc
@then("a ToolCheckpointRequiredError should be raised")
def step_check_checkpoint_required(context: Context) -> None:
assert isinstance(context.checkpoint_error, ToolCheckpointRequiredError)
# ── Cancellation ─────────────────────────────────────────────────────
@given("I cancel the context cancellation token")
def step_cancel_context_token(context: Context) -> None:
context.tool_ctx.cancellation_token.cancel()
@when('I try to execute tool "{name}" after cancellation')
def step_try_execute_cancelled(context: Context, name: str) -> None:
try:
context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx)
context.cancel_error = None
except ToolCancelledError as exc:
context.cancel_error = exc
@then("a ToolCancelledError should be raised from execution")
def step_check_cancelled_error(context: Context) -> None:
assert isinstance(context.cancel_error, ToolCancelledError)
# ── Deactivation ─────────────────────────────────────────────────────
@when('I deactivate tool "{name}" in the runtime')
def step_deactivate_tool(context: Context, name: str) -> None:
context.tool_runtime.deactivate(name, context.tool_ctx)
@then("the mock instance should have been deactivated")
def step_check_mock_deactivated(context: Context) -> None:
for mock in context.mock_instances.values():
if mock.deactivated:
return
raise AssertionError("No mock instance was deactivated")
@when('I deactivate plan "{plan_id}" in the runtime')
def step_deactivate_plan(context: Context, plan_id: str) -> None:
context.tool_runtime.deactivate_plan(context.tool_ctx)
@then("all mock instances should have been deactivated")
def step_check_all_deactivated(context: Context) -> None:
for mock in context.mock_instances.values():
assert mock.deactivated, f"Mock {mock.name} was not deactivated"
# ── Unregistered tool ────────────────────────────────────────────────
@when('I try to execute unregistered tool "{name}"')
def step_try_execute_unregistered(context: Context, name: str) -> None:
try:
context.tool_runtime.execute(name, {}, context.tool_ctx)
context.runtime_error = None
except ToolRuntimeError as exc:
context.runtime_error = exc
@then('a ToolRuntimeError should be raised with message containing "{text}"')
def step_check_runtime_error(context: Context, text: str) -> None:
assert context.runtime_error is not None
assert text in str(context.runtime_error)
# ── Cache stats ──────────────────────────────────────────────────────
@then('the cache stats for plan "{plan_id}" should show {count:d} active tools')
def step_check_runtime_cache_stats(context: Context, plan_id: str, count: int) -> None:
stats = context.tool_runtime.get_cache_stats(plan_id)
assert stats["active_tools"] == count
# ── Trace assertions ─────────────────────────────────────────────────
@then("the last trace should have a non-negative duration_ms")
def step_check_trace_duration(context: Context) -> None:
traces = context.tool_ctx.traces
assert len(traces) > 0
last = traces[-1]
assert last.duration_ms is not None
assert last.duration_ms >= 0
# ── ToolExecutionTrace steps ─────────────────────────────────────────
@given('I create a tool execution trace for "{name}"')
def step_create_trace(context: Context, name: str) -> None:
context.trace = ToolExecutionTrace(
tool_name=name,
started_at=datetime.now(UTC).isoformat(),
)
@then('the trace tool_name should be "{expected}"')
def step_check_trace_name(context: Context, expected: str) -> None:
assert context.trace.tool_name == expected
@then("the trace should have a started_at timestamp")
def step_check_trace_started(context: Context) -> None:
assert context.trace.started_at is not None
datetime.fromisoformat(context.trace.started_at)
# ---------------------------------------------------------------------------
# Scenario: Activation failure raises ToolActivationError
# ---------------------------------------------------------------------------
@given('I register a builtin tool "{name}" with a failing activate mock')
def step_register_failing_activate(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = FailingActivateMock(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@when('I try to activate the failing tool "{name}"')
def step_try_activate_failing(context: Context, name: str) -> None:
try:
context.tool_runtime.activate(name, context.tool_ctx)
context.activation_error = None
except ToolActivationError as exc:
context.activation_error = exc
@then("a ToolActivationError should be raised")
def step_check_activation_error(context: Context) -> None:
assert isinstance(context.activation_error, ToolActivationError)
# ---------------------------------------------------------------------------
# Scenario: Execution failure raises ToolExecutionError
# ---------------------------------------------------------------------------
@given('I register a builtin tool "{name}" with a failing execute mock')
def step_register_failing_execute(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = FailingExecuteMock(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@when('I try to execute the crashing tool "{name}"')
def step_try_execute_crashing(context: Context, name: str) -> None:
try:
context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx)
context.execution_error = None
except ToolExecutionError as exc:
context.execution_error = exc
@then("a ToolExecutionError should be raised")
def step_check_execution_error(context: Context) -> None:
assert isinstance(context.execution_error, ToolExecutionError)
@then("the last trace should show failure")
def step_check_last_trace_failure(context: Context) -> None:
traces = context.tool_ctx.traces
assert len(traces) > 0
assert not traces[-1].success
# ---------------------------------------------------------------------------
# Scenario: Deactivation failure is non-fatal warning
# ---------------------------------------------------------------------------
@given('I register a builtin tool "{name}" with a failing deactivate mock')
def step_register_failing_deactivate(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = FailingDeactivateMock(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
context.deactivation_exception = None
@then("no exception should have been raised from deactivation")
def step_check_no_deactivation_exception(context: Context) -> None:
# If we got here, no exception was raised (deactivation is non-fatal)
assert context.deactivation_exception is None
# ---------------------------------------------------------------------------
# Scenario: Cache plan_count property tracks plans
# ---------------------------------------------------------------------------
@then("the runtime cache plan count should be {count:d}")
def step_check_runtime_plan_count(context: Context, count: int) -> None:
assert context.tool_runtime.cache_plan_count == count
# ---------------------------------------------------------------------------
# Scenario: Discover unregistered tool raises ToolRuntimeError
# ---------------------------------------------------------------------------
@when('I try to discover unregistered tool "{name}"')
def step_try_discover_unregistered(context: Context, name: str) -> None:
try:
context.tool_runtime.discover(name)
context.runtime_error = None
except ToolRuntimeError as exc:
context.runtime_error = exc
# ---------------------------------------------------------------------------
# Scenario: Execution with result error records trace error field
# ---------------------------------------------------------------------------
@given('I register a builtin tool "{name}" with a mock that returns error result')
def step_register_error_result_mock(context: Context, name: str) -> None:
tool = _make_tool(name, read_only=True)
mock = ErrorResultMock(name=name, read_only=True)
context.tool_runtime.register_tool(tool, mock)
context.mock_instances[name] = mock
@then("the last trace should show the error message")
def step_check_last_trace_error(context: Context) -> None:
traces = context.tool_ctx.traces
assert len(traces) > 0
last = traces[-1]
assert last.error is not None
assert "something went wrong" in last.error