1e987a2009
CI / build (push) Successful in 19s
CI / helm (push) Successful in 22s
CI / security (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / e2e_tests (push) Has been cancelled
CI / quality (push) Has been cancelled
CI / typecheck (push) Has been cancelled
CI / integration_tests (push) Has been cancelled
CI / unit_tests (push) Has been cancelled
CI / benchmark-publish (push) Has been cancelled
CI / benchmark-regression (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
CI / coverage (push) Has been cancelled
Co-authored-by: Luis Mendes <luis.mendes@cleverthis.com> Co-committed-by: Luis Mendes <luis.mendes@cleverthis.com>
1046 lines
37 KiB
Python
1046 lines
37 KiB
Python
"""Step definitions for tool lifecycle runtime feature tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.domain.models.core.tool import (
|
|
Tool,
|
|
ToolCapability,
|
|
ToolSource,
|
|
ToolType,
|
|
)
|
|
from cleveragents.infrastructure.events.reactive import ReactiveEventBus
|
|
from cleveragents.tool.context import (
|
|
BoundResource,
|
|
CancellationToken,
|
|
Change,
|
|
ChangeOperation,
|
|
ToolCancelledError,
|
|
ToolExecutionContext,
|
|
ToolExecutionTrace,
|
|
)
|
|
from cleveragents.tool.lifecycle import (
|
|
ToolAccessDeniedError,
|
|
ToolActivationError,
|
|
ToolCheckpointRequiredError,
|
|
ToolDescriptor,
|
|
ToolExecutionError,
|
|
ToolLifecycleCache,
|
|
ToolResult,
|
|
ToolRuntime,
|
|
ToolRuntimeError,
|
|
)
|
|
from cleveragents.tool.schema_validator import (
|
|
ToolSchemaValidationError,
|
|
validate_tool_input,
|
|
validate_tool_output,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Mock tool instance
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class MockToolInstance:
|
|
"""A mock ToolInstance for testing."""
|
|
|
|
def __init__(self, name: str = "mock/tool", read_only: bool = True) -> None:
|
|
self.name = name
|
|
self.read_only = read_only
|
|
self.activated = False
|
|
self.deactivated = False
|
|
self.activate_count = 0
|
|
self.execute_count = 0
|
|
self.produce_changes = False
|
|
self.return_invalid_output = False
|
|
|
|
def discover(self) -> ToolDescriptor:
|
|
return ToolDescriptor(
|
|
name=self.name,
|
|
description=f"Mock tool {self.name}",
|
|
capability=ToolCapability(read_only=self.read_only),
|
|
)
|
|
|
|
def activate(self, ctx: ToolExecutionContext) -> None:
|
|
self.activated = True
|
|
self.activate_count += 1
|
|
|
|
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
|
|
self.execute_count += 1
|
|
changes: list[Change] = []
|
|
if self.produce_changes:
|
|
changes.append(
|
|
Change(
|
|
operation=ChangeOperation.MODIFY,
|
|
resource_id="res-001",
|
|
path=params.get("path", "unknown"),
|
|
)
|
|
)
|
|
|
|
data: Any = f"result for {params.get('path', 'unknown')}"
|
|
if self.return_invalid_output:
|
|
data = 42 # Intentionally wrong type for schema validation
|
|
|
|
return ToolResult(success=True, data=data, changes=changes)
|
|
|
|
def deactivate(self, ctx: ToolExecutionContext) -> None:
|
|
self.deactivated = True
|
|
|
|
|
|
class FailingActivateMock(MockToolInstance):
|
|
"""Mock that raises on activate()."""
|
|
|
|
def activate(self, ctx: ToolExecutionContext) -> None:
|
|
raise RuntimeError("activate boom")
|
|
|
|
|
|
class FailingExecuteMock(MockToolInstance):
|
|
"""Mock that raises on execute()."""
|
|
|
|
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
|
|
raise RuntimeError("execute boom")
|
|
|
|
|
|
class FailingDeactivateMock(MockToolInstance):
|
|
"""Mock that raises on deactivate()."""
|
|
|
|
def deactivate(self, ctx: ToolExecutionContext) -> None:
|
|
raise RuntimeError("deactivate boom")
|
|
|
|
|
|
class ErrorResultMock(MockToolInstance):
|
|
"""Mock that returns a ToolResult with error set."""
|
|
|
|
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
|
|
return ToolResult(success=False, data=None, error="something went wrong")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolExecutionContext steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I create a tool execution context with plan_id "{plan_id}"')
|
|
def step_create_context(context: Context, plan_id: str) -> None:
|
|
context.tool_ctx = ToolExecutionContext(plan_id=plan_id)
|
|
|
|
|
|
@given('I create a tool execution context with plan_id "{plan_id}" and read_only True')
|
|
def step_create_context_read_only(context: Context, plan_id: str) -> None:
|
|
context.tool_ctx = ToolExecutionContext(plan_id=plan_id, plan_read_only=True)
|
|
|
|
|
|
@given(
|
|
'I create a tool execution context with plan_id "{plan_id}" and require_checkpoints True'
|
|
)
|
|
def step_create_context_checkpoints(context: Context, plan_id: str) -> None:
|
|
context.tool_ctx = ToolExecutionContext(plan_id=plan_id, require_checkpoints=True)
|
|
|
|
|
|
@then('the context plan_id should be "{expected}"')
|
|
def step_check_context_plan_id(context: Context, expected: str) -> None:
|
|
assert context.tool_ctx.plan_id == expected
|
|
|
|
|
|
@then("the context should not be read-only")
|
|
def step_check_context_not_read_only(context: Context) -> None:
|
|
assert not context.tool_ctx.plan_read_only
|
|
|
|
|
|
@then("the context should be read-only")
|
|
def step_check_context_read_only(context: Context) -> None:
|
|
assert context.tool_ctx.plan_read_only
|
|
|
|
|
|
@then("the context should not require checkpoints")
|
|
def step_check_context_no_checkpoints(context: Context) -> None:
|
|
assert not context.tool_ctx.require_checkpoints
|
|
|
|
|
|
@then("the context should require checkpoints")
|
|
def step_check_context_checkpoints(context: Context) -> None:
|
|
assert context.tool_ctx.require_checkpoints
|
|
|
|
|
|
@then("the context should have {count:d} changes")
|
|
def step_check_context_change_count(context: Context, count: int) -> None:
|
|
assert len(context.tool_ctx.changes) == count
|
|
|
|
|
|
@then("the context should have {count:d} traces")
|
|
def step_check_context_trace_count(context: Context, count: int) -> None:
|
|
assert len(context.tool_ctx.traces) == count
|
|
|
|
|
|
@then("the context cancellation token should not be cancelled")
|
|
def step_check_token_not_cancelled(context: Context) -> None:
|
|
assert not context.tool_ctx.cancellation_token.is_cancelled
|
|
|
|
|
|
# ── Change recording ──────────────────────────────────────────────────
|
|
|
|
|
|
@when('I record a change with operation "{op}" and resource_id "{res_id}"')
|
|
def step_record_change(context: Context, op: str, res_id: str) -> None:
|
|
change = Change(operation=ChangeOperation(op), resource_id=res_id)
|
|
context.tool_ctx.record_change(change)
|
|
|
|
|
|
# ── Trace recording ──────────────────────────────────────────────────
|
|
|
|
|
|
@when('I add a trace for tool "{tool_name}" with success {success}')
|
|
def step_add_trace(context: Context, tool_name: str, success: str) -> None:
|
|
trace = ToolExecutionTrace(
|
|
tool_name=tool_name,
|
|
started_at=datetime.now(UTC).isoformat(),
|
|
success=success == "True",
|
|
)
|
|
context.tool_ctx.add_trace(trace)
|
|
|
|
|
|
# ── Resource binding ─────────────────────────────────────────────────
|
|
|
|
|
|
@given('I bind resource "{res_id}" to slot "{slot}" with type "{res_type}"')
|
|
def step_bind_resource(context: Context, res_id: str, slot: str, res_type: str) -> None:
|
|
bound = BoundResource(
|
|
slot_name=slot,
|
|
resource_id=res_id,
|
|
resource_type=res_type,
|
|
)
|
|
context.tool_ctx.resources[slot] = bound
|
|
|
|
|
|
@when('I get the resource for slot "{slot}"')
|
|
def step_get_resource(context: Context, slot: str) -> None:
|
|
context.bound_resource = context.tool_ctx.get_resource(slot)
|
|
|
|
|
|
@then('the bound resource_id should be "{expected}"')
|
|
def step_check_bound_resource_id(context: Context, expected: str) -> None:
|
|
assert context.bound_resource.resource_id == expected
|
|
|
|
|
|
@when('I try to get the resource for slot "{slot}"')
|
|
def step_try_get_resource(context: Context, slot: str) -> None:
|
|
try:
|
|
context.tool_ctx.get_resource(slot)
|
|
context.resource_error = None
|
|
except KeyError as exc:
|
|
context.resource_error = exc
|
|
|
|
|
|
@then('a KeyError should be raised with message containing "{text}"')
|
|
def step_check_key_error(context: Context, text: str) -> None:
|
|
assert context.resource_error is not None
|
|
assert text in str(context.resource_error)
|
|
|
|
|
|
# ── Context summary ──────────────────────────────────────────────────
|
|
|
|
|
|
@then("the context summary should show resource_count {count:d}")
|
|
def step_check_summary_resources(context: Context, count: int) -> None:
|
|
summary = context.tool_ctx.as_summary()
|
|
assert summary["resource_count"] == count
|
|
|
|
|
|
@then("the context summary should show change_count {count:d}")
|
|
def step_check_summary_changes(context: Context, count: int) -> None:
|
|
summary = context.tool_ctx.as_summary()
|
|
assert summary["change_count"] == count
|
|
|
|
|
|
@then("the context summary should show trace_count {count:d}")
|
|
def step_check_summary_traces(context: Context, count: int) -> None:
|
|
summary = context.tool_ctx.as_summary()
|
|
assert summary["trace_count"] == count
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BoundResource steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given(
|
|
'I create a bound resource with slot "{slot}" and resource_id "{res_id}" and type "{res_type}"'
|
|
)
|
|
def step_create_bound_resource(
|
|
context: Context, slot: str, res_id: str, res_type: str
|
|
) -> None:
|
|
context.bound_resource = BoundResource(
|
|
slot_name=slot,
|
|
resource_id=res_id,
|
|
resource_type=res_type,
|
|
)
|
|
|
|
|
|
@then('the bound resource slot_name should be "{expected}"')
|
|
def step_check_bound_slot(context: Context, expected: str) -> None:
|
|
assert context.bound_resource.slot_name == expected
|
|
|
|
|
|
@then('the bound resource access should be "{expected}"')
|
|
def step_check_bound_access(context: Context, expected: str) -> None:
|
|
assert context.bound_resource.access == expected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CancellationToken steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I create a cancellation token")
|
|
def step_create_cancellation_token(context: Context) -> None:
|
|
context.cancel_token = CancellationToken()
|
|
|
|
|
|
@when("I cancel the token")
|
|
def step_cancel_token(context: Context) -> None:
|
|
context.cancel_token.cancel()
|
|
|
|
|
|
@then("the cancellation token should not be cancelled")
|
|
def step_check_token_not_cancelled_standalone(context: Context) -> None:
|
|
assert not context.cancel_token.is_cancelled
|
|
|
|
|
|
@then("the cancellation token should be cancelled")
|
|
def step_check_token_cancelled(context: Context) -> None:
|
|
assert context.cancel_token.is_cancelled
|
|
|
|
|
|
@then("calling check on the token should raise ToolCancelledError")
|
|
def step_check_token_raises(context: Context) -> None:
|
|
try:
|
|
context.cancel_token.check()
|
|
raise AssertionError("Expected ToolCancelledError")
|
|
except ToolCancelledError:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Change steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I create a change with operation "{op}" and resource_id "{res_id}"')
|
|
def step_create_change(context: Context, op: str, res_id: str) -> None:
|
|
context.change = Change(operation=ChangeOperation(op), resource_id=res_id)
|
|
|
|
|
|
@then('the change operation should be "{expected}"')
|
|
def step_check_change_op(context: Context, expected: str) -> None:
|
|
assert context.change.operation == expected
|
|
|
|
|
|
@then('the change resource_id should be "{expected}"')
|
|
def step_check_change_res_id(context: Context, expected: str) -> None:
|
|
assert context.change.resource_id == expected
|
|
|
|
|
|
@then("the change timestamp should be a valid ISO-8601 string")
|
|
def step_check_change_timestamp(context: Context) -> None:
|
|
ts = context.change.timestamp
|
|
assert ts is not None
|
|
# Should parse without error
|
|
datetime.fromisoformat(ts)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolDescriptor steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I create a tool descriptor with name "{name}" and description "{desc}"')
|
|
def step_create_descriptor(context: Context, name: str, desc: str) -> None:
|
|
context.descriptor = ToolDescriptor(name=name, description=desc)
|
|
|
|
|
|
@then('the descriptor name should be "{expected}"')
|
|
def step_check_descriptor_name(context: Context, expected: str) -> None:
|
|
assert context.descriptor.name == expected
|
|
|
|
|
|
@then('the descriptor source should be "{expected}"')
|
|
def step_check_descriptor_source(context: Context, expected: str) -> None:
|
|
assert context.descriptor.source == expected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolResult steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I create a tool result with success {success} and data "{data}"')
|
|
def step_create_result_success(context: Context, success: str, data: str) -> None:
|
|
context.tool_result = ToolResult(success=success == "True", data=data)
|
|
|
|
|
|
@given('I create a tool result with success {success} and error "{error}"')
|
|
def step_create_result_error(context: Context, success: str, error: str) -> None:
|
|
context.tool_result = ToolResult(success=success == "True", error=error)
|
|
|
|
|
|
@then("the tool result should indicate success")
|
|
def step_check_result_success(context: Context) -> None:
|
|
assert context.tool_result.success
|
|
|
|
|
|
@then("the tool result should indicate failure")
|
|
def step_check_result_not_success(context: Context) -> None:
|
|
assert not context.tool_result.success
|
|
|
|
|
|
@then('the tool result data should be "{expected}"')
|
|
def step_check_result_data(context: Context, expected: str) -> None:
|
|
assert context.tool_result.data == expected
|
|
|
|
|
|
@then('the tool result error should be "{expected}"')
|
|
def step_check_result_error(context: Context, expected: str) -> None:
|
|
assert context.tool_result.error == expected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JSON Schema validation steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I have a JSON schema requiring property "{prop}" of type "{prop_type}"')
|
|
def step_create_schema(context: Context, prop: str, prop_type: str) -> None:
|
|
context.json_schema = {
|
|
"type": "object",
|
|
"properties": {prop: {"type": prop_type}},
|
|
"required": [prop],
|
|
}
|
|
|
|
|
|
@when('I validate input with path "{value}" against the schema')
|
|
def step_validate_input_string(context: Context, value: str) -> None:
|
|
try:
|
|
validate_tool_input({"path": value}, context.json_schema)
|
|
context.schema_error = None
|
|
except ToolSchemaValidationError as exc:
|
|
context.schema_error = exc
|
|
|
|
|
|
@when("I validate input with path {value:d} against the schema")
|
|
def step_validate_input_int(context: Context, value: int) -> None:
|
|
try:
|
|
validate_tool_input({"path": value}, context.json_schema)
|
|
context.schema_error = None
|
|
except ToolSchemaValidationError as exc:
|
|
context.schema_error = exc
|
|
|
|
|
|
@then("the input validation should succeed")
|
|
def step_check_input_valid(context: Context) -> None:
|
|
assert context.schema_error is None
|
|
|
|
|
|
@then("the input validation should fail with ToolSchemaValidationError")
|
|
def step_check_input_invalid(context: Context) -> None:
|
|
assert context.schema_error is not None
|
|
assert isinstance(context.schema_error, ToolSchemaValidationError)
|
|
|
|
|
|
@when('I validate output with content "{value}" against the schema')
|
|
def step_validate_output_string(context: Context, value: str) -> None:
|
|
try:
|
|
validate_tool_output({"content": value}, context.json_schema)
|
|
context.schema_error = None
|
|
except ToolSchemaValidationError as exc:
|
|
context.schema_error = exc
|
|
|
|
|
|
@when("I validate output with content {value:d} against the schema")
|
|
def step_validate_output_int(context: Context, value: int) -> None:
|
|
try:
|
|
validate_tool_output({"content": value}, context.json_schema)
|
|
context.schema_error = None
|
|
except ToolSchemaValidationError as exc:
|
|
context.schema_error = exc
|
|
|
|
|
|
@then("the output validation should succeed")
|
|
def step_check_output_valid(context: Context) -> None:
|
|
assert context.schema_error is None
|
|
|
|
|
|
@then("the output validation should fail with ToolSchemaValidationError")
|
|
def step_check_output_invalid(context: Context) -> None:
|
|
assert context.schema_error is not None
|
|
assert isinstance(context.schema_error, ToolSchemaValidationError)
|
|
|
|
|
|
@then("the schema validation error should have errors list")
|
|
def step_check_schema_error_details(context: Context) -> None:
|
|
assert context.schema_error is not None
|
|
assert len(context.schema_error.errors) > 0
|
|
|
|
|
|
@then("the schema validation error should reference the schema")
|
|
def step_check_schema_error_schema(context: Context) -> None:
|
|
assert context.schema_error is not None
|
|
assert context.schema_error.schema is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolLifecycleCache steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I create a tool lifecycle cache")
|
|
def step_create_cache(context: Context) -> None:
|
|
context.lifecycle_cache = ToolLifecycleCache()
|
|
|
|
|
|
@given('I have a mock tool instance named "{name}"')
|
|
def step_create_mock_instance(context: Context, name: str) -> None:
|
|
context.mock_instance = MockToolInstance(name=name)
|
|
|
|
|
|
@when('I put the instance in cache for plan "{plan_id}" and tool "{tool_name}"')
|
|
def step_cache_put(context: Context, plan_id: str, tool_name: str) -> None:
|
|
context.lifecycle_cache.put(plan_id, tool_name, context.mock_instance)
|
|
|
|
|
|
@when('I put the second instance in cache for plan "{plan_id}" and tool "{tool_name}"')
|
|
def step_cache_put_second(context: Context, plan_id: str, tool_name: str) -> None:
|
|
# Use a separate instance if available, otherwise create one
|
|
inst = getattr(context, "mock_instance_2", None)
|
|
if inst is None:
|
|
inst = MockToolInstance(name=tool_name)
|
|
context.mock_instance_2 = inst
|
|
context.lifecycle_cache.put(plan_id, tool_name, inst)
|
|
|
|
|
|
@then("the cache should have {count:d} plans")
|
|
def step_check_cache_plan_count(context: Context, count: int) -> None:
|
|
assert context.lifecycle_cache.plan_count == count
|
|
|
|
|
|
@then(
|
|
'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return it'
|
|
)
|
|
def step_cache_get_returns(context: Context, plan_id: str, tool_name: str) -> None:
|
|
result = context.lifecycle_cache.get(plan_id, tool_name)
|
|
assert result is not None
|
|
|
|
|
|
@then(
|
|
'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return None'
|
|
)
|
|
def step_cache_get_returns_none(context: Context, plan_id: str, tool_name: str) -> None:
|
|
result = context.lifecycle_cache.get(plan_id, tool_name)
|
|
assert result is None
|
|
|
|
|
|
@when('I increment execution for plan "{plan_id}" and tool "{tool_name}"')
|
|
def step_cache_increment(context: Context, plan_id: str, tool_name: str) -> None:
|
|
context.lifecycle_cache.increment_execution(plan_id, tool_name)
|
|
|
|
|
|
@then(
|
|
'the cache stats for plan "{plan_id}" should show tool "{tool_name}" with execution_count {count:d}'
|
|
)
|
|
def step_check_cache_stats_exec(
|
|
context: Context, plan_id: str, tool_name: str, count: int
|
|
) -> None:
|
|
stats = context.lifecycle_cache.get_stats(plan_id)
|
|
assert stats["tools"][tool_name]["execution_count"] == count
|
|
|
|
|
|
@then('the plan tools for plan "{plan_id}" should include "{tool_name}"')
|
|
def step_check_plan_tools_include(
|
|
context: Context, plan_id: str, tool_name: str
|
|
) -> None:
|
|
tools = context.lifecycle_cache.get_plan_tools(plan_id)
|
|
assert tool_name in tools
|
|
|
|
|
|
@when('I remove tool "{tool_name}" from plan "{plan_id}"')
|
|
def step_cache_remove_tool(context: Context, tool_name: str, plan_id: str) -> None:
|
|
context.lifecycle_cache.remove(plan_id, tool_name)
|
|
|
|
|
|
@when('I remove plan "{plan_id}" from cache')
|
|
def step_cache_remove_plan(context: Context, plan_id: str) -> None:
|
|
context.lifecycle_cache.remove_plan(plan_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolRuntime steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_tool(
|
|
name: str,
|
|
*,
|
|
read_only: bool = True,
|
|
writes: bool = False,
|
|
checkpointable: bool = False,
|
|
input_schema: dict[str, Any] | None = None,
|
|
output_schema: dict[str, Any] | None = None,
|
|
) -> Tool:
|
|
"""Create a minimal Tool domain model for tests."""
|
|
return Tool(
|
|
name=name,
|
|
description=f"Test tool {name}",
|
|
source=ToolSource.BUILTIN,
|
|
tool_type=ToolType.TOOL,
|
|
capability=ToolCapability(
|
|
read_only=read_only,
|
|
writes=writes,
|
|
checkpointable=checkpointable,
|
|
),
|
|
input_schema=input_schema,
|
|
output_schema=output_schema,
|
|
)
|
|
|
|
|
|
@given("I create a tool runtime")
|
|
def step_create_runtime(context: Context) -> None:
|
|
context.tool_runtime = ToolRuntime(event_bus=ReactiveEventBus())
|
|
context.mock_instances = dict[str, MockToolInstance]()
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a mock instance')
|
|
def step_register_tool(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=False)
|
|
mock = MockToolInstance(name=name, read_only=False)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a mock instance that is read_only')
|
|
def step_register_read_only_tool(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = MockToolInstance(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given('I register a builtin tool "{name}" that writes')
|
|
def step_register_writes_tool(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=False, writes=True)
|
|
mock = MockToolInstance(name=name, read_only=False)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given('I register a builtin tool "{name}" that produces changes')
|
|
def step_register_change_tool(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=False, writes=True)
|
|
mock = MockToolInstance(name=name, read_only=False)
|
|
mock.produce_changes = True
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given('I register a builtin tool with input schema "{name}"')
|
|
def step_register_tool_with_input_schema(context: Context, name: str) -> None:
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"path": {"type": "string"}},
|
|
"required": ["path"],
|
|
}
|
|
tool = _make_tool(name, read_only=True, input_schema=schema)
|
|
mock = MockToolInstance(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given('I register a builtin tool with output schema "{name}"')
|
|
def step_register_tool_with_output_schema(context: Context, name: str) -> None:
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"result": {"type": "string"}},
|
|
"required": ["result"],
|
|
}
|
|
tool = _make_tool(name, read_only=True, output_schema=schema)
|
|
mock = MockToolInstance(name=name, read_only=True)
|
|
mock.return_invalid_output = True # Will return int instead of dict
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@given(
|
|
'I register a second builtin tool "{name}" with a mock instance that is read_only'
|
|
)
|
|
def step_register_second_tool(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = MockToolInstance(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@when('I unregister tool "{name}"')
|
|
def step_unregister_tool(context: Context, name: str) -> None:
|
|
context.tool_runtime.unregister_tool(name)
|
|
|
|
|
|
@then('the tool list should contain "{name}"')
|
|
def step_check_tool_in_list(context: Context, name: str) -> None:
|
|
assert name in context.tool_runtime.list_tools()
|
|
|
|
|
|
@then('the tool list should not contain "{name}"')
|
|
def step_check_tool_not_in_list(context: Context, name: str) -> None:
|
|
assert name not in context.tool_runtime.list_tools()
|
|
|
|
|
|
@then('getting tool "{name}" should return the tool model')
|
|
def step_check_get_tool(context: Context, name: str) -> None:
|
|
tool = context.tool_runtime.get_tool(name)
|
|
assert tool is not None
|
|
assert tool.name == name
|
|
|
|
|
|
@then('getting tool "{name}" should return None')
|
|
def step_check_get_tool_none(context: Context, name: str) -> None:
|
|
assert context.tool_runtime.get_tool(name) is None
|
|
|
|
|
|
@when('I discover tool "{name}"')
|
|
def step_discover_tool(context: Context, name: str) -> None:
|
|
context.discovered_descriptor = context.tool_runtime.discover(name)
|
|
|
|
|
|
@then('the discovered descriptor name should be "{expected}"')
|
|
def step_check_discovered_name(context: Context, expected: str) -> None:
|
|
assert context.discovered_descriptor.name == expected
|
|
|
|
|
|
@when('I activate tool "{name}" in the runtime')
|
|
def step_activate_tool(context: Context, name: str) -> None:
|
|
context.tool_runtime.activate(name, context.tool_ctx)
|
|
|
|
|
|
@when('I activate tool "{name}" in the runtime again')
|
|
def step_activate_tool_again(context: Context, name: str) -> None:
|
|
context.tool_runtime.activate(name, context.tool_ctx)
|
|
|
|
|
|
@then("the mock instance should have been activated")
|
|
def step_check_mock_activated(context: Context) -> None:
|
|
# Find the first mock instance that was activated
|
|
for mock in context.mock_instances.values():
|
|
if mock.activated:
|
|
return
|
|
raise AssertionError("No mock instance was activated")
|
|
|
|
|
|
@then("the mock instance activate count should be {count:d}")
|
|
def step_check_mock_activate_count(context: Context, count: int) -> None:
|
|
total = sum(m.activate_count for m in context.mock_instances.values())
|
|
assert total == count
|
|
|
|
|
|
@when('I execute tool "{name}" with params path "{path}"')
|
|
def step_execute_tool(context: Context, name: str, path: str) -> None:
|
|
context.exec_result = context.tool_runtime.execute(
|
|
name, {"path": path}, context.tool_ctx
|
|
)
|
|
|
|
|
|
@when('I execute tool "{name}" with invalid params')
|
|
def step_execute_tool_invalid(context: Context, name: str) -> None:
|
|
# Pass integer instead of string for 'path'
|
|
context.exec_result = context.tool_runtime.execute(
|
|
name, {"path": 42}, context.tool_ctx
|
|
)
|
|
|
|
|
|
@then("the execution result should be successful")
|
|
def step_check_exec_success(context: Context) -> None:
|
|
assert context.exec_result.success
|
|
|
|
|
|
@then("the execution result should not be successful")
|
|
def step_check_exec_not_success(context: Context) -> None:
|
|
assert not context.exec_result.success
|
|
|
|
|
|
@then('the execution result error should contain "{text}"')
|
|
def step_check_exec_error_contains(context: Context, text: str) -> None:
|
|
assert context.exec_result.error is not None
|
|
assert text in context.exec_result.error
|
|
|
|
|
|
# ── Access denied / checkpoint required ──────────────────────────────
|
|
|
|
|
|
@when('I try to execute tool "{name}"')
|
|
def step_try_execute_tool(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.execute(name, {}, context.tool_ctx)
|
|
context.access_error = None
|
|
except ToolAccessDeniedError as exc:
|
|
context.access_error = exc
|
|
except ToolCheckpointRequiredError as exc:
|
|
context.access_error = exc
|
|
|
|
|
|
@then("a ToolAccessDeniedError should be raised")
|
|
def step_check_access_denied(context: Context) -> None:
|
|
assert isinstance(context.access_error, ToolAccessDeniedError)
|
|
|
|
|
|
@when('I try to activate tool "{name}" for checkpoint plan')
|
|
def step_try_activate_checkpoint(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.activate(name, context.tool_ctx)
|
|
context.checkpoint_error = None
|
|
except ToolCheckpointRequiredError as exc:
|
|
context.checkpoint_error = exc
|
|
|
|
|
|
@then("a ToolCheckpointRequiredError should be raised")
|
|
def step_check_checkpoint_required(context: Context) -> None:
|
|
assert isinstance(context.checkpoint_error, ToolCheckpointRequiredError)
|
|
|
|
|
|
# ── Cancellation ─────────────────────────────────────────────────────
|
|
|
|
|
|
@given("I cancel the context cancellation token")
|
|
def step_cancel_context_token(context: Context) -> None:
|
|
context.tool_ctx.cancellation_token.cancel()
|
|
|
|
|
|
@when('I try to execute tool "{name}" after cancellation')
|
|
def step_try_execute_cancelled(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx)
|
|
context.cancel_error = None
|
|
except ToolCancelledError as exc:
|
|
context.cancel_error = exc
|
|
|
|
|
|
@then("a ToolCancelledError should be raised from execution")
|
|
def step_check_cancelled_error(context: Context) -> None:
|
|
assert isinstance(context.cancel_error, ToolCancelledError)
|
|
|
|
|
|
# ── Deactivation ─────────────────────────────────────────────────────
|
|
|
|
|
|
@when('I deactivate tool "{name}" in the runtime')
|
|
def step_deactivate_tool(context: Context, name: str) -> None:
|
|
context.tool_runtime.deactivate(name, context.tool_ctx)
|
|
|
|
|
|
@then("the mock instance should have been deactivated")
|
|
def step_check_mock_deactivated(context: Context) -> None:
|
|
for mock in context.mock_instances.values():
|
|
if mock.deactivated:
|
|
return
|
|
raise AssertionError("No mock instance was deactivated")
|
|
|
|
|
|
@when('I deactivate plan "{plan_id}" in the runtime')
|
|
def step_deactivate_plan(context: Context, plan_id: str) -> None:
|
|
context.tool_runtime.deactivate_plan(context.tool_ctx)
|
|
|
|
|
|
@then("all mock instances should have been deactivated")
|
|
def step_check_all_deactivated(context: Context) -> None:
|
|
for mock in context.mock_instances.values():
|
|
assert mock.deactivated, f"Mock {mock.name} was not deactivated"
|
|
|
|
|
|
# ── Unregistered tool ────────────────────────────────────────────────
|
|
|
|
|
|
@when('I try to execute unregistered tool "{name}"')
|
|
def step_try_execute_unregistered(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.execute(name, {}, context.tool_ctx)
|
|
context.runtime_error = None
|
|
except ToolRuntimeError as exc:
|
|
context.runtime_error = exc
|
|
|
|
|
|
@then('a ToolRuntimeError should be raised with message containing "{text}"')
|
|
def step_check_runtime_error(context: Context, text: str) -> None:
|
|
assert context.runtime_error is not None
|
|
assert text in str(context.runtime_error)
|
|
|
|
|
|
# ── Cache stats ──────────────────────────────────────────────────────
|
|
|
|
|
|
@then('the cache stats for plan "{plan_id}" should show {count:d} active tools')
|
|
def step_check_runtime_cache_stats(context: Context, plan_id: str, count: int) -> None:
|
|
stats = context.tool_runtime.get_cache_stats(plan_id)
|
|
assert stats["active_tools"] == count
|
|
|
|
|
|
# ── Trace assertions ─────────────────────────────────────────────────
|
|
|
|
|
|
@then("the last trace should have a non-negative duration_ms")
|
|
def step_check_trace_duration(context: Context) -> None:
|
|
traces = context.tool_ctx.traces
|
|
assert len(traces) > 0
|
|
last = traces[-1]
|
|
assert last.duration_ms is not None
|
|
assert last.duration_ms >= 0
|
|
|
|
|
|
# ── ToolExecutionTrace steps ─────────────────────────────────────────
|
|
|
|
|
|
@given('I create a tool execution trace for "{name}"')
|
|
def step_create_trace(context: Context, name: str) -> None:
|
|
context.trace = ToolExecutionTrace(
|
|
tool_name=name,
|
|
started_at=datetime.now(UTC).isoformat(),
|
|
)
|
|
|
|
|
|
@then('the trace tool_name should be "{expected}"')
|
|
def step_check_trace_name(context: Context, expected: str) -> None:
|
|
assert context.trace.tool_name == expected
|
|
|
|
|
|
@then("the trace should have a started_at timestamp")
|
|
def step_check_trace_started(context: Context) -> None:
|
|
assert context.trace.started_at is not None
|
|
datetime.fromisoformat(context.trace.started_at)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Activation failure raises ToolActivationError
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a failing activate mock')
|
|
def step_register_failing_activate(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = FailingActivateMock(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@when('I try to activate the failing tool "{name}"')
|
|
def step_try_activate_failing(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.activate(name, context.tool_ctx)
|
|
context.activation_error = None
|
|
except ToolActivationError as exc:
|
|
context.activation_error = exc
|
|
|
|
|
|
@then("a ToolActivationError should be raised")
|
|
def step_check_activation_error(context: Context) -> None:
|
|
assert isinstance(context.activation_error, ToolActivationError)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Execution failure raises ToolExecutionError
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a failing execute mock')
|
|
def step_register_failing_execute(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = FailingExecuteMock(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@when('I try to execute the crashing tool "{name}"')
|
|
def step_try_execute_crashing(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx)
|
|
context.execution_error = None
|
|
except ToolExecutionError as exc:
|
|
context.execution_error = exc
|
|
|
|
|
|
@then("a ToolExecutionError should be raised")
|
|
def step_check_execution_error(context: Context) -> None:
|
|
assert isinstance(context.execution_error, ToolExecutionError)
|
|
|
|
|
|
@then("the last trace should show failure")
|
|
def step_check_last_trace_failure(context: Context) -> None:
|
|
traces = context.tool_ctx.traces
|
|
assert len(traces) > 0
|
|
assert not traces[-1].success
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Deactivation failure is non-fatal warning
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a failing deactivate mock')
|
|
def step_register_failing_deactivate(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = FailingDeactivateMock(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
context.deactivation_exception = None
|
|
|
|
|
|
@then("no exception should have been raised from deactivation")
|
|
def step_check_no_deactivation_exception(context: Context) -> None:
|
|
# If we got here, no exception was raised (deactivation is non-fatal)
|
|
assert context.deactivation_exception is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Cache plan_count property tracks plans
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the runtime cache plan count should be {count:d}")
|
|
def step_check_runtime_plan_count(context: Context, count: int) -> None:
|
|
assert context.tool_runtime.cache_plan_count == count
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Discover unregistered tool raises ToolRuntimeError
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I try to discover unregistered tool "{name}"')
|
|
def step_try_discover_unregistered(context: Context, name: str) -> None:
|
|
try:
|
|
context.tool_runtime.discover(name)
|
|
context.runtime_error = None
|
|
except ToolRuntimeError as exc:
|
|
context.runtime_error = exc
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Execution with result error records trace error field
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I register a builtin tool "{name}" with a mock that returns error result')
|
|
def step_register_error_result_mock(context: Context, name: str) -> None:
|
|
tool = _make_tool(name, read_only=True)
|
|
mock = ErrorResultMock(name=name, read_only=True)
|
|
context.tool_runtime.register_tool(tool, mock)
|
|
context.mock_instances[name] = mock
|
|
|
|
|
|
@then("the last trace should show the error message")
|
|
def step_check_last_trace_error(context: Context) -> None:
|
|
traces = context.tool_ctx.traces
|
|
assert len(traces) > 0
|
|
last = traces[-1]
|
|
assert last.error is not None
|
|
assert "something went wrong" in last.error
|