diff --git a/benchmarks/tool_lifecycle_bench.py b/benchmarks/tool_lifecycle_bench.py new file mode 100644 index 000000000..82d76c2f5 --- /dev/null +++ b/benchmarks/tool_lifecycle_bench.py @@ -0,0 +1,165 @@ +"""ASV benchmarks for tool lifecycle runtime.""" + +from __future__ import annotations + +from typing import Any + +from cleveragents.tool.context import ( + BoundResource, + CancellationToken, + Change, + ChangeOperation, + ToolExecutionContext, +) +from cleveragents.tool.lifecycle import ( + ToolDescriptor, + ToolLifecycleCache, + ToolResult, +) +from cleveragents.tool.schema_validator import validate_tool_input + + +class _DummyInstance: + """Minimal mock for benchmarks.""" + + def discover(self) -> ToolDescriptor: + return ToolDescriptor(name="bench/tool", description="bench", source="builtin") + + def activate(self, ctx: Any) -> None: + pass + + def execute(self, params: Any, ctx: Any) -> ToolResult: + return ToolResult() + + def deactivate(self, ctx: Any) -> None: + pass + + +class TimeToolExecutionContext: + """Benchmark ToolExecutionContext creation and operations.""" + + def time_create_context(self) -> None: + ToolExecutionContext(plan_id="plan-bench-001") + + def time_create_context_with_resources(self) -> None: + resources = { + f"slot_{i}": BoundResource( + slot_name=f"slot_{i}", + resource_id=f"res-{i:03d}", + resource_type="git-checkout", + access="read_write", + ) + for i in range(10) + } + ToolExecutionContext( + plan_id="plan-bench-002", + resources=resources, + ) + + def time_record_change(self) -> None: + ctx = ToolExecutionContext(plan_id="plan-bench-003") + for i in range(100): + ctx.record_change( + Change( + operation=ChangeOperation.MODIFY, + resource_id=f"res-{i:03d}", + ) + ) + + def time_context_summary(self) -> None: + ctx = ToolExecutionContext(plan_id="plan-bench-004") + for _ in range(50): + ctx.as_summary() + + +class TimeCancellationToken: + """Benchmark CancellationToken operations.""" + + def time_create_token(self) -> None: + CancellationToken() + + def time_check_not_cancelled(self) -> None: + token = CancellationToken() + for _ in range(1000): + _ = token.is_cancelled + + def time_cancel_and_check(self) -> None: + token = CancellationToken() + token.cancel() + for _ in range(1000): + _ = token.is_cancelled + + +class TimeToolLifecycleCache: + """Benchmark ToolLifecycleCache operations.""" + + def time_put_and_get(self) -> None: + cache = ToolLifecycleCache() + dummy = _DummyInstance() + for i in range(100): + cache.put(f"plan-{i}", f"tool/t-{i}", dummy) + for i in range(100): + cache.get(f"plan-{i}", f"tool/t-{i}") + + def time_remove_plan(self) -> None: + cache = ToolLifecycleCache() + dummy = _DummyInstance() + for i in range(50): + cache.put("plan-001", f"tool/t-{i}", dummy) + cache.remove_plan("plan-001") + + def time_get_stats(self) -> None: + cache = ToolLifecycleCache() + dummy = _DummyInstance() + for i in range(20): + cache.put("plan-001", f"tool/t-{i}", dummy) + for _ in range(50): + cache.get_stats("plan-001") + + +class TimeSchemaValidation: + """Benchmark JSON Schema validation.""" + + def setup(self) -> None: + self.schema: dict[str, Any] = { + "type": "object", + "properties": { + "path": {"type": "string"}, + "content": {"type": "string"}, + "mode": {"type": "integer"}, + }, + "required": ["path"], + } + + def time_validate_simple(self) -> None: + validate_tool_input({"path": "test.txt"}, self.schema) + + def time_validate_full(self) -> None: + validate_tool_input( + {"path": "test.txt", "content": "hello", "mode": 644}, + self.schema, + ) + + +class TimeToolDescriptor: + """Benchmark ToolDescriptor creation.""" + + def time_create_descriptor(self) -> None: + for i in range(100): + ToolDescriptor( + name=f"ns/tool-{i}", + description=f"Tool {i}", + ) + + def time_create_descriptor_with_schema(self) -> None: + schema: dict[str, Any] = { + "type": "object", + "properties": {"path": {"type": "string"}}, + } + for i in range(100): + ToolDescriptor( + name=f"ns/tool-{i}", + description=f"Tool {i}", + input_schema=schema, + output_schema=schema, + ) diff --git a/docs/reference/tool_lifecycle.md b/docs/reference/tool_lifecycle.md new file mode 100644 index 000000000..bdf994cd7 --- /dev/null +++ b/docs/reference/tool_lifecycle.md @@ -0,0 +1,104 @@ +# Tool Lifecycle Runtime + +The tool lifecycle runtime orchestrates the four-stage lifecycle +(`discover`/`activate`/`execute`/`deactivate`) for all tool sources +(MCP, Agent Skills, built-ins, custom, wrapped). + +## Architecture + +``` +ToolRuntime + ├── ToolLifecycleCache (per-plan activation reuse) + ├── ToolInstance (Protocol) (concrete tool implementations) + ├── ToolExecutionContext (plan metadata + resources + cancellation) + └── schema_validator (JSON Schema input/output validation) +``` + +## Four-Stage Lifecycle + +### 1. Discover + +Returns lightweight `ToolDescriptor` metadata without activating the tool. +Used during registration and tool enumeration. + +### 2. Activate + +Prepares the tool for execution within a plan context. Called at most once +per plan via the lifecycle cache. May start external processes (MCP), load +instructions (Agent Skills), or no-op (built-ins). + +### 3. Execute + +Runs the tool with validated parameters. The runtime: +1. Checks cancellation +2. Auto-activates if not yet activated for the plan +3. Enforces capability flags (read-only, checkpoint requirements) +4. Validates inputs against JSON Schema +5. Executes the tool with tracing +6. Validates outputs against JSON Schema +7. Records changes into the execution context + +### 4. Deactivate + +Cleans up after all executions. Called once per plan on completion, failure, +or cancellation. Must be idempotent and must not raise. + +## Capability Enforcement + +| Constraint | Condition | Error | +|-----------|-----------|-------| +| Read-only plan | Tool has `writes=True` | `ToolAccessDeniedError` | +| Checkpoint required | Tool has `checkpointable=False` | `ToolCheckpointRequiredError` | + +## Per-Plan Activation Cache + +The `ToolLifecycleCache` ensures: +- Each tool is activated at most once per plan +- `deactivate()` is guaranteed for every activated tool when the plan ends +- Thread-safe via `RLock` for concurrent plan execution + +## Cancellation + +The `CancellationToken` propagates cancellation from the plan lifecycle: +- Set via `cancel()` when `agents plan cancel` is invoked +- Tools check `ctx.cancellation_token.is_cancelled` for long-running ops +- `ctx.cancellation_token.check()` raises `ToolCancelledError` immediately + +## JSON Schema Validation + +Uses JSON Schema draft 2020-12 via `jsonschema.Draft202012Validator`: +- `input_schema`: validated before execution; failures return error result +- `output_schema`: validated after execution; failures logged as warning + +## Execution Tracing + +Every execution produces a `ToolExecutionTrace` with: +- `started_at` / `ended_at` (ISO-8601) +- `duration_ms` +- `result_size_bytes` +- `success` / `error` + +## Error Hierarchy + +``` +ToolRuntimeError + ├── ToolAccessDeniedError + ├── ToolCheckpointRequiredError + ├── ToolNotActivatedError + ├── ToolActivationError + ├── ToolExecutionError + └── ToolDeactivationError + +ToolCancelledError (standalone, not ToolRuntimeError) +ToolSchemaValidationError (standalone) +``` + +## Key Files + +| File | Purpose | +|------|---------| +| `src/cleveragents/tool/__init__.py` | Package exports | +| `src/cleveragents/tool/context.py` | `ToolExecutionContext`, `BoundResource`, `Change`, `CancellationToken`, `ToolExecutionTrace` | +| `src/cleveragents/tool/lifecycle.py` | `ToolRuntime`, `ToolInstance`, `ToolLifecycleCache`, `ToolDescriptor`, `ToolResult` | +| `src/cleveragents/tool/schema_validator.py` | JSON Schema validation for inputs/outputs | +| `src/cleveragents/domain/models/core/tool.py` | `Tool`, `Validation`, `ToolCapability` domain models | diff --git a/features/steps/tool_lifecycle_runtime_steps.py b/features/steps/tool_lifecycle_runtime_steps.py new file mode 100644 index 000000000..d2db68438 --- /dev/null +++ b/features/steps/tool_lifecycle_runtime_steps.py @@ -0,0 +1,1044 @@ +"""Step definitions for tool lifecycle runtime feature tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.domain.models.core.tool import ( + Tool, + ToolCapability, + ToolSource, + ToolType, +) +from cleveragents.tool.context import ( + BoundResource, + CancellationToken, + Change, + ChangeOperation, + ToolCancelledError, + ToolExecutionContext, + ToolExecutionTrace, +) +from cleveragents.tool.lifecycle import ( + ToolAccessDeniedError, + ToolActivationError, + ToolCheckpointRequiredError, + ToolDescriptor, + ToolExecutionError, + ToolLifecycleCache, + ToolResult, + ToolRuntime, + ToolRuntimeError, +) +from cleveragents.tool.schema_validator import ( + ToolSchemaValidationError, + validate_tool_input, + validate_tool_output, +) + +# --------------------------------------------------------------------------- +# Mock tool instance +# --------------------------------------------------------------------------- + + +class MockToolInstance: + """A mock ToolInstance for testing.""" + + def __init__(self, name: str = "mock/tool", read_only: bool = True) -> None: + self.name = name + self.read_only = read_only + self.activated = False + self.deactivated = False + self.activate_count = 0 + self.execute_count = 0 + self.produce_changes = False + self.return_invalid_output = False + + def discover(self) -> ToolDescriptor: + return ToolDescriptor( + name=self.name, + description=f"Mock tool {self.name}", + capability=ToolCapability(read_only=self.read_only), + ) + + def activate(self, ctx: ToolExecutionContext) -> None: + self.activated = True + self.activate_count += 1 + + def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult: + self.execute_count += 1 + changes: list[Change] = [] + if self.produce_changes: + changes.append( + Change( + operation=ChangeOperation.MODIFY, + resource_id="res-001", + path=params.get("path", "unknown"), + ) + ) + + data: Any = f"result for {params.get('path', 'unknown')}" + if self.return_invalid_output: + data = 42 # Intentionally wrong type for schema validation + + return ToolResult(success=True, data=data, changes=changes) + + def deactivate(self, ctx: ToolExecutionContext) -> None: + self.deactivated = True + + +class FailingActivateMock(MockToolInstance): + """Mock that raises on activate().""" + + def activate(self, ctx: ToolExecutionContext) -> None: + raise RuntimeError("activate boom") + + +class FailingExecuteMock(MockToolInstance): + """Mock that raises on execute().""" + + def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult: + raise RuntimeError("execute boom") + + +class FailingDeactivateMock(MockToolInstance): + """Mock that raises on deactivate().""" + + def deactivate(self, ctx: ToolExecutionContext) -> None: + raise RuntimeError("deactivate boom") + + +class ErrorResultMock(MockToolInstance): + """Mock that returns a ToolResult with error set.""" + + def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult: + return ToolResult(success=False, data=None, error="something went wrong") + + +# --------------------------------------------------------------------------- +# ToolExecutionContext steps +# --------------------------------------------------------------------------- + + +@given('I create a tool execution context with plan_id "{plan_id}"') +def step_create_context(context: Context, plan_id: str) -> None: + context.tool_ctx = ToolExecutionContext(plan_id=plan_id) + + +@given('I create a tool execution context with plan_id "{plan_id}" and read_only True') +def step_create_context_read_only(context: Context, plan_id: str) -> None: + context.tool_ctx = ToolExecutionContext(plan_id=plan_id, plan_read_only=True) + + +@given( + 'I create a tool execution context with plan_id "{plan_id}" and require_checkpoints True' +) +def step_create_context_checkpoints(context: Context, plan_id: str) -> None: + context.tool_ctx = ToolExecutionContext(plan_id=plan_id, require_checkpoints=True) + + +@then('the context plan_id should be "{expected}"') +def step_check_context_plan_id(context: Context, expected: str) -> None: + assert context.tool_ctx.plan_id == expected + + +@then("the context should not be read-only") +def step_check_context_not_read_only(context: Context) -> None: + assert not context.tool_ctx.plan_read_only + + +@then("the context should be read-only") +def step_check_context_read_only(context: Context) -> None: + assert context.tool_ctx.plan_read_only + + +@then("the context should not require checkpoints") +def step_check_context_no_checkpoints(context: Context) -> None: + assert not context.tool_ctx.require_checkpoints + + +@then("the context should require checkpoints") +def step_check_context_checkpoints(context: Context) -> None: + assert context.tool_ctx.require_checkpoints + + +@then("the context should have {count:d} changes") +def step_check_context_change_count(context: Context, count: int) -> None: + assert len(context.tool_ctx.changes) == count + + +@then("the context should have {count:d} traces") +def step_check_context_trace_count(context: Context, count: int) -> None: + assert len(context.tool_ctx.traces) == count + + +@then("the context cancellation token should not be cancelled") +def step_check_token_not_cancelled(context: Context) -> None: + assert not context.tool_ctx.cancellation_token.is_cancelled + + +# ── Change recording ────────────────────────────────────────────────── + + +@when('I record a change with operation "{op}" and resource_id "{res_id}"') +def step_record_change(context: Context, op: str, res_id: str) -> None: + change = Change(operation=ChangeOperation(op), resource_id=res_id) + context.tool_ctx.record_change(change) + + +# ── Trace recording ────────────────────────────────────────────────── + + +@when('I add a trace for tool "{tool_name}" with success {success}') +def step_add_trace(context: Context, tool_name: str, success: str) -> None: + trace = ToolExecutionTrace( + tool_name=tool_name, + started_at=datetime.now(UTC).isoformat(), + success=success == "True", + ) + context.tool_ctx.add_trace(trace) + + +# ── Resource binding ───────────────────────────────────────────────── + + +@given('I bind resource "{res_id}" to slot "{slot}" with type "{res_type}"') +def step_bind_resource(context: Context, res_id: str, slot: str, res_type: str) -> None: + bound = BoundResource( + slot_name=slot, + resource_id=res_id, + resource_type=res_type, + ) + context.tool_ctx.resources[slot] = bound + + +@when('I get the resource for slot "{slot}"') +def step_get_resource(context: Context, slot: str) -> None: + context.bound_resource = context.tool_ctx.get_resource(slot) + + +@then('the bound resource_id should be "{expected}"') +def step_check_bound_resource_id(context: Context, expected: str) -> None: + assert context.bound_resource.resource_id == expected + + +@when('I try to get the resource for slot "{slot}"') +def step_try_get_resource(context: Context, slot: str) -> None: + try: + context.tool_ctx.get_resource(slot) + context.resource_error = None + except KeyError as exc: + context.resource_error = exc + + +@then('a KeyError should be raised with message containing "{text}"') +def step_check_key_error(context: Context, text: str) -> None: + assert context.resource_error is not None + assert text in str(context.resource_error) + + +# ── Context summary ────────────────────────────────────────────────── + + +@then("the context summary should show resource_count {count:d}") +def step_check_summary_resources(context: Context, count: int) -> None: + summary = context.tool_ctx.as_summary() + assert summary["resource_count"] == count + + +@then("the context summary should show change_count {count:d}") +def step_check_summary_changes(context: Context, count: int) -> None: + summary = context.tool_ctx.as_summary() + assert summary["change_count"] == count + + +@then("the context summary should show trace_count {count:d}") +def step_check_summary_traces(context: Context, count: int) -> None: + summary = context.tool_ctx.as_summary() + assert summary["trace_count"] == count + + +# --------------------------------------------------------------------------- +# BoundResource steps +# --------------------------------------------------------------------------- + + +@given( + 'I create a bound resource with slot "{slot}" and resource_id "{res_id}" and type "{res_type}"' +) +def step_create_bound_resource( + context: Context, slot: str, res_id: str, res_type: str +) -> None: + context.bound_resource = BoundResource( + slot_name=slot, + resource_id=res_id, + resource_type=res_type, + ) + + +@then('the bound resource slot_name should be "{expected}"') +def step_check_bound_slot(context: Context, expected: str) -> None: + assert context.bound_resource.slot_name == expected + + +@then('the bound resource access should be "{expected}"') +def step_check_bound_access(context: Context, expected: str) -> None: + assert context.bound_resource.access == expected + + +# --------------------------------------------------------------------------- +# CancellationToken steps +# --------------------------------------------------------------------------- + + +@given("I create a cancellation token") +def step_create_cancellation_token(context: Context) -> None: + context.cancel_token = CancellationToken() + + +@when("I cancel the token") +def step_cancel_token(context: Context) -> None: + context.cancel_token.cancel() + + +@then("the cancellation token should not be cancelled") +def step_check_token_not_cancelled_standalone(context: Context) -> None: + assert not context.cancel_token.is_cancelled + + +@then("the cancellation token should be cancelled") +def step_check_token_cancelled(context: Context) -> None: + assert context.cancel_token.is_cancelled + + +@then("calling check on the token should raise ToolCancelledError") +def step_check_token_raises(context: Context) -> None: + try: + context.cancel_token.check() + raise AssertionError("Expected ToolCancelledError") + except ToolCancelledError: + pass + + +# --------------------------------------------------------------------------- +# Change steps +# --------------------------------------------------------------------------- + + +@given('I create a change with operation "{op}" and resource_id "{res_id}"') +def step_create_change(context: Context, op: str, res_id: str) -> None: + context.change = Change(operation=ChangeOperation(op), resource_id=res_id) + + +@then('the change operation should be "{expected}"') +def step_check_change_op(context: Context, expected: str) -> None: + assert context.change.operation == expected + + +@then('the change resource_id should be "{expected}"') +def step_check_change_res_id(context: Context, expected: str) -> None: + assert context.change.resource_id == expected + + +@then("the change timestamp should be a valid ISO-8601 string") +def step_check_change_timestamp(context: Context) -> None: + ts = context.change.timestamp + assert ts is not None + # Should parse without error + datetime.fromisoformat(ts) + + +# --------------------------------------------------------------------------- +# ToolDescriptor steps +# --------------------------------------------------------------------------- + + +@given('I create a tool descriptor with name "{name}" and description "{desc}"') +def step_create_descriptor(context: Context, name: str, desc: str) -> None: + context.descriptor = ToolDescriptor(name=name, description=desc) + + +@then('the descriptor name should be "{expected}"') +def step_check_descriptor_name(context: Context, expected: str) -> None: + assert context.descriptor.name == expected + + +@then('the descriptor source should be "{expected}"') +def step_check_descriptor_source(context: Context, expected: str) -> None: + assert context.descriptor.source == expected + + +# --------------------------------------------------------------------------- +# ToolResult steps +# --------------------------------------------------------------------------- + + +@given('I create a tool result with success {success} and data "{data}"') +def step_create_result_success(context: Context, success: str, data: str) -> None: + context.tool_result = ToolResult(success=success == "True", data=data) + + +@given('I create a tool result with success {success} and error "{error}"') +def step_create_result_error(context: Context, success: str, error: str) -> None: + context.tool_result = ToolResult(success=success == "True", error=error) + + +@then("the tool result should indicate success") +def step_check_result_success(context: Context) -> None: + assert context.tool_result.success + + +@then("the tool result should indicate failure") +def step_check_result_not_success(context: Context) -> None: + assert not context.tool_result.success + + +@then('the tool result data should be "{expected}"') +def step_check_result_data(context: Context, expected: str) -> None: + assert context.tool_result.data == expected + + +@then('the tool result error should be "{expected}"') +def step_check_result_error(context: Context, expected: str) -> None: + assert context.tool_result.error == expected + + +# --------------------------------------------------------------------------- +# JSON Schema validation steps +# --------------------------------------------------------------------------- + + +@given('I have a JSON schema requiring property "{prop}" of type "{prop_type}"') +def step_create_schema(context: Context, prop: str, prop_type: str) -> None: + context.json_schema = { + "type": "object", + "properties": {prop: {"type": prop_type}}, + "required": [prop], + } + + +@when('I validate input with path "{value}" against the schema') +def step_validate_input_string(context: Context, value: str) -> None: + try: + validate_tool_input({"path": value}, context.json_schema) + context.schema_error = None + except ToolSchemaValidationError as exc: + context.schema_error = exc + + +@when("I validate input with path {value:d} against the schema") +def step_validate_input_int(context: Context, value: int) -> None: + try: + validate_tool_input({"path": value}, context.json_schema) + context.schema_error = None + except ToolSchemaValidationError as exc: + context.schema_error = exc + + +@then("the input validation should succeed") +def step_check_input_valid(context: Context) -> None: + assert context.schema_error is None + + +@then("the input validation should fail with ToolSchemaValidationError") +def step_check_input_invalid(context: Context) -> None: + assert context.schema_error is not None + assert isinstance(context.schema_error, ToolSchemaValidationError) + + +@when('I validate output with content "{value}" against the schema') +def step_validate_output_string(context: Context, value: str) -> None: + try: + validate_tool_output({"content": value}, context.json_schema) + context.schema_error = None + except ToolSchemaValidationError as exc: + context.schema_error = exc + + +@when("I validate output with content {value:d} against the schema") +def step_validate_output_int(context: Context, value: int) -> None: + try: + validate_tool_output({"content": value}, context.json_schema) + context.schema_error = None + except ToolSchemaValidationError as exc: + context.schema_error = exc + + +@then("the output validation should succeed") +def step_check_output_valid(context: Context) -> None: + assert context.schema_error is None + + +@then("the output validation should fail with ToolSchemaValidationError") +def step_check_output_invalid(context: Context) -> None: + assert context.schema_error is not None + assert isinstance(context.schema_error, ToolSchemaValidationError) + + +@then("the schema validation error should have errors list") +def step_check_schema_error_details(context: Context) -> None: + assert context.schema_error is not None + assert len(context.schema_error.errors) > 0 + + +@then("the schema validation error should reference the schema") +def step_check_schema_error_schema(context: Context) -> None: + assert context.schema_error is not None + assert context.schema_error.schema is not None + + +# --------------------------------------------------------------------------- +# ToolLifecycleCache steps +# --------------------------------------------------------------------------- + + +@given("I create a tool lifecycle cache") +def step_create_cache(context: Context) -> None: + context.lifecycle_cache = ToolLifecycleCache() + + +@given('I have a mock tool instance named "{name}"') +def step_create_mock_instance(context: Context, name: str) -> None: + context.mock_instance = MockToolInstance(name=name) + + +@when('I put the instance in cache for plan "{plan_id}" and tool "{tool_name}"') +def step_cache_put(context: Context, plan_id: str, tool_name: str) -> None: + context.lifecycle_cache.put(plan_id, tool_name, context.mock_instance) + + +@when('I put the second instance in cache for plan "{plan_id}" and tool "{tool_name}"') +def step_cache_put_second(context: Context, plan_id: str, tool_name: str) -> None: + # Use a separate instance if available, otherwise create one + inst = getattr(context, "mock_instance_2", None) + if inst is None: + inst = MockToolInstance(name=tool_name) + context.mock_instance_2 = inst + context.lifecycle_cache.put(plan_id, tool_name, inst) + + +@then("the cache should have {count:d} plans") +def step_check_cache_plan_count(context: Context, count: int) -> None: + assert context.lifecycle_cache.plan_count == count + + +@then( + 'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return it' +) +def step_cache_get_returns(context: Context, plan_id: str, tool_name: str) -> None: + result = context.lifecycle_cache.get(plan_id, tool_name) + assert result is not None + + +@then( + 'getting the instance for plan "{plan_id}" and tool "{tool_name}" should return None' +) +def step_cache_get_returns_none(context: Context, plan_id: str, tool_name: str) -> None: + result = context.lifecycle_cache.get(plan_id, tool_name) + assert result is None + + +@when('I increment execution for plan "{plan_id}" and tool "{tool_name}"') +def step_cache_increment(context: Context, plan_id: str, tool_name: str) -> None: + context.lifecycle_cache.increment_execution(plan_id, tool_name) + + +@then( + 'the cache stats for plan "{plan_id}" should show tool "{tool_name}" with execution_count {count:d}' +) +def step_check_cache_stats_exec( + context: Context, plan_id: str, tool_name: str, count: int +) -> None: + stats = context.lifecycle_cache.get_stats(plan_id) + assert stats["tools"][tool_name]["execution_count"] == count + + +@then('the plan tools for plan "{plan_id}" should include "{tool_name}"') +def step_check_plan_tools_include( + context: Context, plan_id: str, tool_name: str +) -> None: + tools = context.lifecycle_cache.get_plan_tools(plan_id) + assert tool_name in tools + + +@when('I remove tool "{tool_name}" from plan "{plan_id}"') +def step_cache_remove_tool(context: Context, tool_name: str, plan_id: str) -> None: + context.lifecycle_cache.remove(plan_id, tool_name) + + +@when('I remove plan "{plan_id}" from cache') +def step_cache_remove_plan(context: Context, plan_id: str) -> None: + context.lifecycle_cache.remove_plan(plan_id) + + +# --------------------------------------------------------------------------- +# ToolRuntime steps +# --------------------------------------------------------------------------- + + +def _make_tool( + name: str, + *, + read_only: bool = True, + writes: bool = False, + checkpointable: bool = False, + input_schema: dict[str, Any] | None = None, + output_schema: dict[str, Any] | None = None, +) -> Tool: + """Create a minimal Tool domain model for tests.""" + return Tool( + name=name, + description=f"Test tool {name}", + source=ToolSource.BUILTIN, + tool_type=ToolType.TOOL, + capability=ToolCapability( + read_only=read_only, + writes=writes, + checkpointable=checkpointable, + ), + input_schema=input_schema, + output_schema=output_schema, + ) + + +@given("I create a tool runtime") +def step_create_runtime(context: Context) -> None: + context.tool_runtime = ToolRuntime() + context.mock_instances = dict[str, MockToolInstance]() + + +@given('I register a builtin tool "{name}" with a mock instance') +def step_register_tool(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=False) + mock = MockToolInstance(name=name, read_only=False) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given('I register a builtin tool "{name}" with a mock instance that is read_only') +def step_register_read_only_tool(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = MockToolInstance(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given('I register a builtin tool "{name}" that writes') +def step_register_writes_tool(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=False, writes=True) + mock = MockToolInstance(name=name, read_only=False) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given('I register a builtin tool "{name}" that produces changes') +def step_register_change_tool(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=False, writes=True) + mock = MockToolInstance(name=name, read_only=False) + mock.produce_changes = True + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given('I register a builtin tool with input schema "{name}"') +def step_register_tool_with_input_schema(context: Context, name: str) -> None: + schema = { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + } + tool = _make_tool(name, read_only=True, input_schema=schema) + mock = MockToolInstance(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given('I register a builtin tool with output schema "{name}"') +def step_register_tool_with_output_schema(context: Context, name: str) -> None: + schema = { + "type": "object", + "properties": {"result": {"type": "string"}}, + "required": ["result"], + } + tool = _make_tool(name, read_only=True, output_schema=schema) + mock = MockToolInstance(name=name, read_only=True) + mock.return_invalid_output = True # Will return int instead of dict + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@given( + 'I register a second builtin tool "{name}" with a mock instance that is read_only' +) +def step_register_second_tool(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = MockToolInstance(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@when('I unregister tool "{name}"') +def step_unregister_tool(context: Context, name: str) -> None: + context.tool_runtime.unregister_tool(name) + + +@then('the tool list should contain "{name}"') +def step_check_tool_in_list(context: Context, name: str) -> None: + assert name in context.tool_runtime.list_tools() + + +@then('the tool list should not contain "{name}"') +def step_check_tool_not_in_list(context: Context, name: str) -> None: + assert name not in context.tool_runtime.list_tools() + + +@then('getting tool "{name}" should return the tool model') +def step_check_get_tool(context: Context, name: str) -> None: + tool = context.tool_runtime.get_tool(name) + assert tool is not None + assert tool.name == name + + +@then('getting tool "{name}" should return None') +def step_check_get_tool_none(context: Context, name: str) -> None: + assert context.tool_runtime.get_tool(name) is None + + +@when('I discover tool "{name}"') +def step_discover_tool(context: Context, name: str) -> None: + context.discovered_descriptor = context.tool_runtime.discover(name) + + +@then('the discovered descriptor name should be "{expected}"') +def step_check_discovered_name(context: Context, expected: str) -> None: + assert context.discovered_descriptor.name == expected + + +@when('I activate tool "{name}" in the runtime') +def step_activate_tool(context: Context, name: str) -> None: + context.tool_runtime.activate(name, context.tool_ctx) + + +@when('I activate tool "{name}" in the runtime again') +def step_activate_tool_again(context: Context, name: str) -> None: + context.tool_runtime.activate(name, context.tool_ctx) + + +@then("the mock instance should have been activated") +def step_check_mock_activated(context: Context) -> None: + # Find the first mock instance that was activated + for mock in context.mock_instances.values(): + if mock.activated: + return + raise AssertionError("No mock instance was activated") + + +@then("the mock instance activate count should be {count:d}") +def step_check_mock_activate_count(context: Context, count: int) -> None: + total = sum(m.activate_count for m in context.mock_instances.values()) + assert total == count + + +@when('I execute tool "{name}" with params path "{path}"') +def step_execute_tool(context: Context, name: str, path: str) -> None: + context.exec_result = context.tool_runtime.execute( + name, {"path": path}, context.tool_ctx + ) + + +@when('I execute tool "{name}" with invalid params') +def step_execute_tool_invalid(context: Context, name: str) -> None: + # Pass integer instead of string for 'path' + context.exec_result = context.tool_runtime.execute( + name, {"path": 42}, context.tool_ctx + ) + + +@then("the execution result should be successful") +def step_check_exec_success(context: Context) -> None: + assert context.exec_result.success + + +@then("the execution result should not be successful") +def step_check_exec_not_success(context: Context) -> None: + assert not context.exec_result.success + + +@then('the execution result error should contain "{text}"') +def step_check_exec_error_contains(context: Context, text: str) -> None: + assert context.exec_result.error is not None + assert text in context.exec_result.error + + +# ── Access denied / checkpoint required ────────────────────────────── + + +@when('I try to execute tool "{name}"') +def step_try_execute_tool(context: Context, name: str) -> None: + try: + context.tool_runtime.execute(name, {}, context.tool_ctx) + context.access_error = None + except ToolAccessDeniedError as exc: + context.access_error = exc + except ToolCheckpointRequiredError as exc: + context.access_error = exc + + +@then("a ToolAccessDeniedError should be raised") +def step_check_access_denied(context: Context) -> None: + assert isinstance(context.access_error, ToolAccessDeniedError) + + +@when('I try to activate tool "{name}" for checkpoint plan') +def step_try_activate_checkpoint(context: Context, name: str) -> None: + try: + context.tool_runtime.activate(name, context.tool_ctx) + context.checkpoint_error = None + except ToolCheckpointRequiredError as exc: + context.checkpoint_error = exc + + +@then("a ToolCheckpointRequiredError should be raised") +def step_check_checkpoint_required(context: Context) -> None: + assert isinstance(context.checkpoint_error, ToolCheckpointRequiredError) + + +# ── Cancellation ───────────────────────────────────────────────────── + + +@given("I cancel the context cancellation token") +def step_cancel_context_token(context: Context) -> None: + context.tool_ctx.cancellation_token.cancel() + + +@when('I try to execute tool "{name}" after cancellation') +def step_try_execute_cancelled(context: Context, name: str) -> None: + try: + context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx) + context.cancel_error = None + except ToolCancelledError as exc: + context.cancel_error = exc + + +@then("a ToolCancelledError should be raised from execution") +def step_check_cancelled_error(context: Context) -> None: + assert isinstance(context.cancel_error, ToolCancelledError) + + +# ── Deactivation ───────────────────────────────────────────────────── + + +@when('I deactivate tool "{name}" in the runtime') +def step_deactivate_tool(context: Context, name: str) -> None: + context.tool_runtime.deactivate(name, context.tool_ctx) + + +@then("the mock instance should have been deactivated") +def step_check_mock_deactivated(context: Context) -> None: + for mock in context.mock_instances.values(): + if mock.deactivated: + return + raise AssertionError("No mock instance was deactivated") + + +@when('I deactivate plan "{plan_id}" in the runtime') +def step_deactivate_plan(context: Context, plan_id: str) -> None: + context.tool_runtime.deactivate_plan(context.tool_ctx) + + +@then("all mock instances should have been deactivated") +def step_check_all_deactivated(context: Context) -> None: + for mock in context.mock_instances.values(): + assert mock.deactivated, f"Mock {mock.name} was not deactivated" + + +# ── Unregistered tool ──────────────────────────────────────────────── + + +@when('I try to execute unregistered tool "{name}"') +def step_try_execute_unregistered(context: Context, name: str) -> None: + try: + context.tool_runtime.execute(name, {}, context.tool_ctx) + context.runtime_error = None + except ToolRuntimeError as exc: + context.runtime_error = exc + + +@then('a ToolRuntimeError should be raised with message containing "{text}"') +def step_check_runtime_error(context: Context, text: str) -> None: + assert context.runtime_error is not None + assert text in str(context.runtime_error) + + +# ── Cache stats ────────────────────────────────────────────────────── + + +@then('the cache stats for plan "{plan_id}" should show {count:d} active tools') +def step_check_runtime_cache_stats(context: Context, plan_id: str, count: int) -> None: + stats = context.tool_runtime.get_cache_stats(plan_id) + assert stats["active_tools"] == count + + +# ── Trace assertions ───────────────────────────────────────────────── + + +@then("the last trace should have a non-negative duration_ms") +def step_check_trace_duration(context: Context) -> None: + traces = context.tool_ctx.traces + assert len(traces) > 0 + last = traces[-1] + assert last.duration_ms is not None + assert last.duration_ms >= 0 + + +# ── ToolExecutionTrace steps ───────────────────────────────────────── + + +@given('I create a tool execution trace for "{name}"') +def step_create_trace(context: Context, name: str) -> None: + context.trace = ToolExecutionTrace( + tool_name=name, + started_at=datetime.now(UTC).isoformat(), + ) + + +@then('the trace tool_name should be "{expected}"') +def step_check_trace_name(context: Context, expected: str) -> None: + assert context.trace.tool_name == expected + + +@then("the trace should have a started_at timestamp") +def step_check_trace_started(context: Context) -> None: + assert context.trace.started_at is not None + datetime.fromisoformat(context.trace.started_at) + + +# --------------------------------------------------------------------------- +# Scenario: Activation failure raises ToolActivationError +# --------------------------------------------------------------------------- + + +@given('I register a builtin tool "{name}" with a failing activate mock') +def step_register_failing_activate(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = FailingActivateMock(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@when('I try to activate the failing tool "{name}"') +def step_try_activate_failing(context: Context, name: str) -> None: + try: + context.tool_runtime.activate(name, context.tool_ctx) + context.activation_error = None + except ToolActivationError as exc: + context.activation_error = exc + + +@then("a ToolActivationError should be raised") +def step_check_activation_error(context: Context) -> None: + assert isinstance(context.activation_error, ToolActivationError) + + +# --------------------------------------------------------------------------- +# Scenario: Execution failure raises ToolExecutionError +# --------------------------------------------------------------------------- + + +@given('I register a builtin tool "{name}" with a failing execute mock') +def step_register_failing_execute(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = FailingExecuteMock(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@when('I try to execute the crashing tool "{name}"') +def step_try_execute_crashing(context: Context, name: str) -> None: + try: + context.tool_runtime.execute(name, {"path": "test.txt"}, context.tool_ctx) + context.execution_error = None + except ToolExecutionError as exc: + context.execution_error = exc + + +@then("a ToolExecutionError should be raised") +def step_check_execution_error(context: Context) -> None: + assert isinstance(context.execution_error, ToolExecutionError) + + +@then("the last trace should show failure") +def step_check_last_trace_failure(context: Context) -> None: + traces = context.tool_ctx.traces + assert len(traces) > 0 + assert not traces[-1].success + + +# --------------------------------------------------------------------------- +# Scenario: Deactivation failure is non-fatal warning +# --------------------------------------------------------------------------- + + +@given('I register a builtin tool "{name}" with a failing deactivate mock') +def step_register_failing_deactivate(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = FailingDeactivateMock(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + context.deactivation_exception = None + + +@then("no exception should have been raised from deactivation") +def step_check_no_deactivation_exception(context: Context) -> None: + # If we got here, no exception was raised (deactivation is non-fatal) + assert context.deactivation_exception is None + + +# --------------------------------------------------------------------------- +# Scenario: Cache plan_count property tracks plans +# --------------------------------------------------------------------------- + + +@then("the runtime cache plan count should be {count:d}") +def step_check_runtime_plan_count(context: Context, count: int) -> None: + assert context.tool_runtime.cache_plan_count == count + + +# --------------------------------------------------------------------------- +# Scenario: Discover unregistered tool raises ToolRuntimeError +# --------------------------------------------------------------------------- + + +@when('I try to discover unregistered tool "{name}"') +def step_try_discover_unregistered(context: Context, name: str) -> None: + try: + context.tool_runtime.discover(name) + context.runtime_error = None + except ToolRuntimeError as exc: + context.runtime_error = exc + + +# --------------------------------------------------------------------------- +# Scenario: Execution with result error records trace error field +# --------------------------------------------------------------------------- + + +@given('I register a builtin tool "{name}" with a mock that returns error result') +def step_register_error_result_mock(context: Context, name: str) -> None: + tool = _make_tool(name, read_only=True) + mock = ErrorResultMock(name=name, read_only=True) + context.tool_runtime.register_tool(tool, mock) + context.mock_instances[name] = mock + + +@then("the last trace should show the error message") +def step_check_last_trace_error(context: Context) -> None: + traces = context.tool_ctx.traces + assert len(traces) > 0 + last = traces[-1] + assert last.error is not None + assert "something went wrong" in last.error diff --git a/features/tool_lifecycle_runtime.feature b/features/tool_lifecycle_runtime.feature new file mode 100644 index 000000000..60c84107d --- /dev/null +++ b/features/tool_lifecycle_runtime.feature @@ -0,0 +1,379 @@ +Feature: Tool lifecycle runtime + As the tool execution engine + I need to orchestrate discover/activate/execute/deactivate lifecycle + With capability enforcement, schema validation, caching, and cancellation + + # ── ToolExecutionContext ────────────────────────────────────────────── + + Scenario: Create a tool execution context with defaults + Given I create a tool execution context with plan_id "plan-001" + Then the context plan_id should be "plan-001" + And the context should not be read-only + And the context should not require checkpoints + And the context should have 0 changes + And the context should have 0 traces + And the context cancellation token should not be cancelled + + Scenario: Create a read-only execution context + Given I create a tool execution context with plan_id "plan-002" and read_only True + Then the context should be read-only + + Scenario: Create a checkpoint-required execution context + Given I create a tool execution context with plan_id "plan-003" and require_checkpoints True + Then the context should require checkpoints + + Scenario: Record changes in execution context + Given I create a tool execution context with plan_id "plan-004" + When I record a change with operation "create" and resource_id "res-001" + Then the context should have 1 changes + + Scenario: Add traces to execution context + Given I create a tool execution context with plan_id "plan-005" + When I add a trace for tool "test/tool" with success True + Then the context should have 1 traces + + Scenario: Get resource from context by slot name + Given I create a tool execution context with plan_id "plan-006" + And I bind resource "res-001" to slot "repo" with type "git-checkout" + When I get the resource for slot "repo" + Then the bound resource_id should be "res-001" + + Scenario: Get resource from context with missing slot raises KeyError + Given I create a tool execution context with plan_id "plan-007" + When I try to get the resource for slot "missing" + Then a KeyError should be raised with message containing "missing" + + Scenario: Context summary includes correct counts + Given I create a tool execution context with plan_id "plan-008" + And I bind resource "res-001" to slot "repo" with type "git-checkout" + When I record a change with operation "modify" and resource_id "res-001" + And I add a trace for tool "test/tool" with success True + Then the context summary should show resource_count 1 + And the context summary should show change_count 1 + And the context summary should show trace_count 1 + + # ── BoundResource ──────────────────────────────────────────────────── + + Scenario: Create a BoundResource with all fields + Given I create a bound resource with slot "db" and resource_id "res-db-001" and type "postgresql" + Then the bound resource slot_name should be "db" + And the bound resource access should be "read_only" + + # ── CancellationToken ──────────────────────────────────────────────── + + Scenario: Cancellation token starts not cancelled + Given I create a cancellation token + Then the cancellation token should not be cancelled + + Scenario: Cancel sets the cancellation flag + Given I create a cancellation token + When I cancel the token + Then the cancellation token should be cancelled + + Scenario: Check raises ToolCancelledError when cancelled + Given I create a cancellation token + When I cancel the token + Then calling check on the token should raise ToolCancelledError + + # ── Change tracking ────────────────────────────────────────────────── + + Scenario: Create a change with all operations + Given I create a change with operation "create" and resource_id "res-001" + Then the change operation should be "create" + And the change resource_id should be "res-001" + + Scenario: Change has automatic timestamp + Given I create a change with operation "delete" and resource_id "res-002" + Then the change timestamp should be a valid ISO-8601 string + + # ── ToolDescriptor ─────────────────────────────────────────────────── + + Scenario: Create a tool descriptor + Given I create a tool descriptor with name "builtin/read-file" and description "Read a file" + Then the descriptor name should be "builtin/read-file" + And the descriptor source should be "builtin" + + # ── ToolResult ─────────────────────────────────────────────────────── + + Scenario: Create a successful tool result + Given I create a tool result with success True and data "hello" + Then the tool result should indicate success + And the tool result data should be "hello" + + Scenario: Create a failed tool result + Given I create a tool result with success False and error "something broke" + Then the tool result should indicate failure + And the tool result error should be "something broke" + + # ── JSON Schema validation ─────────────────────────────────────────── + + Scenario: Validate tool input against valid schema + Given I have a JSON schema requiring property "path" of type "string" + When I validate input with path "test.txt" against the schema + Then the input validation should succeed + + Scenario: Validate tool input against invalid data + Given I have a JSON schema requiring property "path" of type "string" + When I validate input with path 42 against the schema + Then the input validation should fail with ToolSchemaValidationError + + Scenario: Validate tool output against valid schema + Given I have a JSON schema requiring property "content" of type "string" + When I validate output with content "file data" against the schema + Then the output validation should succeed + + Scenario: Validate tool output against invalid data + Given I have a JSON schema requiring property "content" of type "string" + When I validate output with content 123 against the schema + Then the output validation should fail with ToolSchemaValidationError + + Scenario: Schema validation error contains error details + Given I have a JSON schema requiring property "path" of type "string" + When I validate input with path 42 against the schema + Then the schema validation error should have errors list + And the schema validation error should reference the schema + + # ── ToolLifecycleCache ─────────────────────────────────────────────── + + Scenario: Cache starts empty + Given I create a tool lifecycle cache + Then the cache should have 0 plans + + Scenario: Put and get an instance from cache + Given I create a tool lifecycle cache + And I have a mock tool instance named "builtin/read" + When I put the instance in cache for plan "plan-001" and tool "builtin/read" + Then getting the instance for plan "plan-001" and tool "builtin/read" should return it + + Scenario: Get returns None for uncached tool + Given I create a tool lifecycle cache + Then getting the instance for plan "plan-001" and tool "builtin/missing" should return None + + Scenario: Increment execution count + Given I create a tool lifecycle cache + And I have a mock tool instance named "builtin/read" + When I put the instance in cache for plan "plan-001" and tool "builtin/read" + And I increment execution for plan "plan-001" and tool "builtin/read" + Then the cache stats for plan "plan-001" should show tool "builtin/read" with execution_count 1 + + Scenario: Get plan tools lists activated tools + Given I create a tool lifecycle cache + And I have a mock tool instance named "builtin/read" + And I have a mock tool instance named "builtin/write" + When I put the instance in cache for plan "plan-001" and tool "builtin/read" + And I put the second instance in cache for plan "plan-001" and tool "builtin/write" + Then the plan tools for plan "plan-001" should include "builtin/read" + And the plan tools for plan "plan-001" should include "builtin/write" + + Scenario: Remove a single tool from cache + Given I create a tool lifecycle cache + And I have a mock tool instance named "builtin/read" + When I put the instance in cache for plan "plan-001" and tool "builtin/read" + And I remove tool "builtin/read" from plan "plan-001" + Then getting the instance for plan "plan-001" and tool "builtin/read" should return None + + Scenario: Remove plan clears all tools + Given I create a tool lifecycle cache + And I have a mock tool instance named "builtin/read" + And I have a mock tool instance named "builtin/write" + When I put the instance in cache for plan "plan-001" and tool "builtin/read" + And I put the second instance in cache for plan "plan-001" and tool "builtin/write" + And I remove plan "plan-001" from cache + Then the cache should have 0 plans + + # ── ToolRuntime ────────────────────────────────────────────────────── + + Scenario: Register and list tools + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance + Then the tool list should contain "builtin/read-file" + + Scenario: Unregister a tool + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance + When I unregister tool "builtin/read-file" + Then the tool list should not contain "builtin/read-file" + + Scenario: Get a registered tool + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance + Then getting tool "builtin/read-file" should return the tool model + + Scenario: Get an unregistered tool returns None + Given I create a tool runtime + Then getting tool "builtin/missing" should return None + + Scenario: Discover returns a ToolDescriptor + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance + When I discover tool "builtin/read-file" + Then the discovered descriptor name should be "builtin/read-file" + + Scenario: Activate a tool for a plan + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-010" + When I activate tool "builtin/read-file" in the runtime + Then the mock instance should have been activated + + Scenario: Activate is idempotent for same plan + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-011" + When I activate tool "builtin/read-file" in the runtime + And I activate tool "builtin/read-file" in the runtime again + Then the mock instance activate count should be 1 + + Scenario: Execute a tool successfully + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-012" + When I execute tool "builtin/read-file" with params path "test.txt" + Then the execution result should be successful + And the context should have 1 traces + + Scenario: Execute a tool with input schema validation failure + Given I create a tool runtime + And I register a builtin tool with input schema "builtin/typed-tool" + And I create a tool execution context with plan_id "plan-013" + When I execute tool "builtin/typed-tool" with invalid params + Then the execution result should not be successful + And the execution result error should contain "Input validation failed" + + Scenario: Read-only plan rejects tool with writes + Given I create a tool runtime + And I register a builtin tool "builtin/write-file" that writes + And I create a tool execution context with plan_id "plan-014" and read_only True + When I try to execute tool "builtin/write-file" + Then a ToolAccessDeniedError should be raised + + Scenario: Checkpoint-required plan rejects non-checkpointable tool + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-015" and require_checkpoints True + When I try to activate tool "builtin/read-file" for checkpoint plan + Then a ToolCheckpointRequiredError should be raised + + Scenario: Execute records changes from tool result + Given I create a tool runtime + And I register a builtin tool "builtin/edit-file" that produces changes + And I create a tool execution context with plan_id "plan-016" + When I execute tool "builtin/edit-file" with params path "test.txt" + Then the context should have 1 changes + + Scenario: Execute with cancelled token raises ToolCancelledError + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-017" + And I cancel the context cancellation token + When I try to execute tool "builtin/read-file" after cancellation + Then a ToolCancelledError should be raised from execution + + Scenario: Deactivate a single tool + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-018" + When I activate tool "builtin/read-file" in the runtime + And I deactivate tool "builtin/read-file" in the runtime + Then the mock instance should have been deactivated + + Scenario: Deactivate plan clears all activated tools + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I register a second builtin tool "builtin/list-files" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-019" + When I activate tool "builtin/read-file" in the runtime + And I activate tool "builtin/list-files" in the runtime + And I deactivate plan "plan-019" in the runtime + Then all mock instances should have been deactivated + + Scenario: Execute unregistered tool raises ToolRuntimeError + Given I create a tool runtime + And I create a tool execution context with plan_id "plan-020" + When I try to execute unregistered tool "builtin/ghost" + Then a ToolRuntimeError should be raised with message containing "not registered" + + Scenario: Cache stats after execution + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-021" + When I execute tool "builtin/read-file" with params path "test.txt" + Then the cache stats for plan "plan-021" should show 1 active tools + + Scenario: Output schema validation failure returns error result + Given I create a tool runtime + And I register a builtin tool with output schema "builtin/validated-output" + And I create a tool execution context with plan_id "plan-022" + When I execute tool "builtin/validated-output" with params path "test.txt" + Then the execution result should not be successful + And the execution result error should contain "Output validation failed" + + Scenario: Execution trace captures duration + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-023" + When I execute tool "builtin/read-file" with params path "test.txt" + Then the last trace should have a non-negative duration_ms + + Scenario: ToolExecutionTrace fields + Given I create a tool execution trace for "builtin/read" + Then the trace tool_name should be "builtin/read" + And the trace should have a started_at timestamp + + Scenario: Read-only plan allows read-only tools + Given I create a tool runtime + And I register a builtin tool "builtin/search" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-024" and read_only True + When I execute tool "builtin/search" with params path "*.py" + Then the execution result should be successful + + Scenario: Activation failure raises ToolActivationError + Given I create a tool runtime + And I register a builtin tool "builtin/broken" with a failing activate mock + And I create a tool execution context with plan_id "plan-025" + When I try to activate the failing tool "builtin/broken" + Then a ToolActivationError should be raised + + Scenario: Execution failure raises ToolExecutionError + Given I create a tool runtime + And I register a builtin tool "builtin/crashing" with a failing execute mock + And I create a tool execution context with plan_id "plan-026" + When I try to execute the crashing tool "builtin/crashing" + Then a ToolExecutionError should be raised + And the context should have 1 traces + And the last trace should show failure + + Scenario: Deactivation failure is non-fatal warning + Given I create a tool runtime + And I register a builtin tool "builtin/sticky" with a failing deactivate mock + And I create a tool execution context with plan_id "plan-027" + When I activate tool "builtin/sticky" in the runtime + And I deactivate tool "builtin/sticky" in the runtime + Then no exception should have been raised from deactivation + + Scenario: Plan deactivation handles failing tools gracefully + Given I create a tool runtime + And I register a builtin tool "builtin/sticky" with a failing deactivate mock + And I create a tool execution context with plan_id "plan-028" + When I activate tool "builtin/sticky" in the runtime + And I deactivate plan "plan-028" in the runtime + Then no exception should have been raised from deactivation + + Scenario: Cache plan_count property tracks plans + Given I create a tool runtime + And I register a builtin tool "builtin/read-file" with a mock instance that is read_only + And I create a tool execution context with plan_id "plan-029" + When I activate tool "builtin/read-file" in the runtime + Then the runtime cache plan count should be 1 + + Scenario: Discover unregistered tool raises ToolRuntimeError + Given I create a tool runtime + When I try to discover unregistered tool "builtin/ghost" + Then a ToolRuntimeError should be raised with message containing "No implementation registered" + + Scenario: Execution with result error records trace error field + Given I create a tool runtime + And I register a builtin tool "builtin/err-tool" with a mock that returns error result + And I create a tool execution context with plan_id "plan-030" + When I execute tool "builtin/err-tool" with params path "test.txt" + Then the last trace should show the error message diff --git a/implementation_plan.md b/implementation_plan.md index a28db0e4b..33fb5c881 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -2738,31 +2738,31 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled **Parallel Group C5: Tool Lifecycle Runtime [Jeff]** (M2; depends on C0.runtime + C1.tool.domain) -- [ ] **COMMIT (Owner: Jeff | Group: C5.lifecycle | Branch: feature/m2-tool-runtime | Planned: Day 11 | Expected: Day 13) - Commit message: "feat(tool): add tool lifecycle runtime"** - - [ ] Git [Jeff]: `git checkout master` - - [ ] Git [Jeff]: `git pull origin master` - - [ ] Git [Jeff]: `git checkout -b feature/m2-tool-runtime` - - [ ] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit) - - [ ] Code [Jeff]: Implement `ToolRuntime`/`ToolInstance` interfaces with `discover/activate/execute/deactivate` hooks and lifecycle state tracking. - - [ ] Code [Jeff]: Add `ToolExecutionContext` with resolved resource bindings, sandbox paths, plan metadata, and cancellation token. - - [ ] Code [Jeff]: Add lifecycle cache with per-plan activation reuse and guaranteed `deactivate` on plan completion/cancel. - - [ ] Code [Jeff]: Enforce tool capability flags (read-only/writes/checkpointable) and read-only plan gating at runtime. - - [ ] Code [Jeff]: Validate tool inputs/outputs against JSON schema before/after execution; surface schema errors clearly. - - [ ] Code [Jeff]: Add tool execution tracing (start/end timestamps, duration, result size) for diagnostics. - - [ ] Code [Jeff]: Add cancellation propagation so long-running tools are interrupted on plan cancel. - - [ ] Docs [Jeff]: Add `docs/reference/tool_lifecycle.md` describing hook ordering, capability enforcement, and failure handling. - - [ ] Docs [Jeff]: Document schema validation behavior and error payload format for tool failures. - - [ ] Tests (Behave) [Jeff]: Add lifecycle scenarios for activate/execute/deactivate ordering and error propagation. - - [ ] Tests (Robot) [Jeff]: Add `robot/tool_lifecycle.robot` runtime smoke tests. - - [ ] Tests (ASV) [Jeff]: Add `benchmarks/tool_lifecycle_bench.py` for lifecycle overhead. - - [ ] Quality [Jeff]: Run `nox` (all default sessions, including benchmark). - - [ ] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. - - [ ] Git [Jeff]: `git add .` (only after nox passes) - - [ ] Commit [Jeff]: `git commit -m "feat(tool): add tool lifecycle runtime"`. - - [ ] Forgejo PR [Jeff]: Open PR from `feature/m2-tool-runtime` to `master` with description "Add tool lifecycle runtime, context, and tests.". - - [ ] Git [Jeff]: `git checkout master` - - [ ] Git [Jeff]: `git branch -d feature/m2-tool-runtime` - - [ ] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. +- [X] **COMMIT (Owner: Jeff | Group: C5.lifecycle | Branch: feature/m2-tool-runtime | Planned: Day 11 | Expected: Day 13) - Commit message: "feat(tool): add tool lifecycle runtime"** + - [X] Git [Jeff]: `git checkout master` + - [X] Git [Jeff]: `git pull origin master` + - [X] Git [Jeff]: `git checkout -b feature/m2-tool-runtime` + - [X] Git [Jeff]: `git fetch origin && git merge origin/master` (run before final tests and before commit) + - [X] Code [Jeff]: Implement `ToolRuntime`/`ToolInstance` interfaces with `discover/activate/execute/deactivate` hooks and lifecycle state tracking. + - [X] Code [Jeff]: Add `ToolExecutionContext` with resolved resource bindings, sandbox paths, plan metadata, and cancellation token. + - [X] Code [Jeff]: Add lifecycle cache with per-plan activation reuse and guaranteed `deactivate` on plan completion/cancel. + - [X] Code [Jeff]: Enforce tool capability flags (read-only/writes/checkpointable) and read-only plan gating at runtime. + - [X] Code [Jeff]: Validate tool inputs/outputs against JSON schema before/after execution; surface schema errors clearly. + - [X] Code [Jeff]: Add tool execution tracing (start/end timestamps, duration, result size) for diagnostics. + - [X] Code [Jeff]: Add cancellation propagation so long-running tools are interrupted on plan cancel. + - [X] Docs [Jeff]: Add `docs/reference/tool_lifecycle.md` describing hook ordering, capability enforcement, and failure handling. + - [X] Docs [Jeff]: Document schema validation behavior and error payload format for tool failures. + - [X] Tests (Behave) [Jeff]: Add lifecycle scenarios for activate/execute/deactivate ordering and error propagation. + - [X] Tests (Robot) [Jeff]: Add `robot/tool_lifecycle.robot` runtime smoke tests. + - [X] Tests (ASV) [Jeff]: Add `benchmarks/tool_lifecycle_bench.py` for lifecycle overhead. + - [X] Quality [Jeff]: Run `nox` (all default sessions, including benchmark). + - [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. + - [X] Git [Jeff]: `git add .` (only after nox passes) + - [X] Commit [Jeff]: `git commit -m "feat(tool): add tool lifecycle runtime"`. + - [X] Forgejo PR [Jeff]: Open PR from `feature/m2-tool-runtime` to `master` with description "Add tool lifecycle runtime, context, and tests.". + - [X] Git [Jeff]: `git checkout master` + - [X] Git [Jeff]: `git branch -d feature/m2-tool-runtime` + - [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. **Parallel Group C1: Actor Schema & Examples [Aditya + Jeff]** (start Day 5; C2 depends on this) - [ ] **COMMIT (Owner: Aditya | Group: C1.schema | Branch: feature/m3-actor-schema-examples | Planned: Day 12 | Expected: Day 14) - Commit message: "feat(actor): add actor yaml schema models"** diff --git a/pyproject.toml b/pyproject.toml index b1e103b85..2e1bb2959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "numpy>=2.1.0", "python-ulid>=2.7.0", # ULID generation for plan/action IDs "RestrictedPython>=7.0", # Secure sandbox for user-supplied code + "jsonschema>=4.20.0", # JSON Schema validation for tool inputs/outputs ] [project.optional-dependencies] diff --git a/robot/tool_lifecycle.robot b/robot/tool_lifecycle.robot new file mode 100644 index 000000000..27d8c8d66 --- /dev/null +++ b/robot/tool_lifecycle.robot @@ -0,0 +1,57 @@ +*** Settings *** +Documentation Tool lifecycle runtime smoke tests +Library Process +Library OperatingSystem + +*** Variables *** +${PYTHON} python + +*** Test Cases *** +Tool Package Is Importable + [Documentation] Verify the tool lifecycle package can be imported + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import ToolRuntime, ToolExecutionContext, ToolResult, ToolDescriptor, ToolLifecycleCache, CancellationToken, Change, ChangeOperation, BoundResource; print('OK') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} OK + +Tool Context Creation Works + [Documentation] Verify ToolExecutionContext can be created + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import ToolExecutionContext; ctx = ToolExecutionContext(plan_id='test-plan'); print(f'plan={ctx.plan_id} changes={len(ctx.changes)} traces={len(ctx.traces)}') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} plan=test-plan + Should Contain ${result.stdout} changes=0 + Should Contain ${result.stdout} traces=0 + +Cancellation Token Works + [Documentation] Verify CancellationToken cancel and check + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import CancellationToken; t = CancellationToken(); print(f'before={t.is_cancelled}'); t.cancel(); print(f'after={t.is_cancelled}') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} before=False + Should Contain ${result.stdout} after=True + +Schema Validator Works + [Documentation] Verify JSON Schema validation + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import validate_tool_input, ToolSchemaValidationError; schema = {"type": "object", "properties": {"x": {"type": "string"}}, "required": ["x"]}; validate_tool_input({"x": "hi"}, schema); print('valid'); ok = False\ntry:\n validate_tool_input({"x": 1}, schema)\nexcept ToolSchemaValidationError:\n ok = True\nprint(f'invalid_caught={ok}') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} valid + Should Contain ${result.stdout} invalid_caught=True + +Tool Runtime Registration Works + [Documentation] Verify ToolRuntime register and list + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import ToolRuntime; from cleveragents.domain.models.core.tool import Tool, ToolSource, ToolCapability; rt = ToolRuntime(); t = Tool(name='test/read', description='Read', source=ToolSource.BUILTIN, capability=ToolCapability(read_only=True)); print(f'before={rt.list_tools()}'); rt.register_tool(t, None); print(f'after={rt.list_tools()}') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} before=[] + Should Contain ${result.stdout} test/read + +Lifecycle Cache Works + [Documentation] Verify ToolLifecycleCache put/get + ${result}= Run Process ${PYTHON} -c + ... from cleveragents.tool import ToolLifecycleCache; c = ToolLifecycleCache(); print(f'plans={c.plan_count}'); c.put('p1', 'tool/a', 'dummy'); print(f'plans_after={c.plan_count}'); r = c.get('p1', 'tool/a'); print(f'got={r}') + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} plans=0 + Should Contain ${result.stdout} plans_after=1 + Should Contain ${result.stdout} got=dummy diff --git a/src/cleveragents/tool/__init__.py b/src/cleveragents/tool/__init__.py index b19cf3ada..a7e6089d0 100644 --- a/src/cleveragents/tool/__init__.py +++ b/src/cleveragents/tool/__init__.py @@ -1,16 +1,76 @@ -"""Tool runtime package for CleverAgents. +"""Tool runtime and lifecycle package for CleverAgents v3. -Re-exports public types from the runtime, runner, and registry modules. +Re-exports public types from the runtime, runner, and registry modules +(C0.runtime) and the four-stage tool lifecycle with capability enforcement, +JSON Schema validation, per-plan activation caching, execution tracing, +and cancellation propagation (C5.lifecycle). """ +# C0.runtime — core tool spec, runner, registry from cleveragents.tool.registry import ToolRegistry from cleveragents.tool.runner import ToolRunner from cleveragents.tool.runtime import ToolError, ToolResult, ToolSpec +# C5.lifecycle — four-stage lifecycle runtime +from cleveragents.tool.context import ( + BoundResource, + CancellationToken, + Change, + ChangeOperation, + ToolCancelledError, + ToolExecutionContext, + ToolExecutionTrace, +) +from cleveragents.tool.lifecycle import ( + ToolAccessDeniedError, + ToolActivationError, + ToolCheckpointRequiredError, + ToolDeactivationError, + ToolDescriptor, + ToolExecutionError, + ToolInstance, + ToolLifecycleCache, + ToolNotActivatedError, + ToolResult as LifecycleToolResult, + ToolRuntime, + ToolRuntimeError, +) +from cleveragents.tool.schema_validator import ( + ToolSchemaValidationError, + validate_tool_input, + validate_tool_output, +) + __all__ = [ + # C0.runtime "ToolError", "ToolRegistry", "ToolResult", "ToolRunner", "ToolSpec", + # C5.lifecycle — context + "BoundResource", + "CancellationToken", + "Change", + "ChangeOperation", + "ToolCancelledError", + "ToolExecutionContext", + "ToolExecutionTrace", + # C5.lifecycle — runtime + "LifecycleToolResult", + "ToolAccessDeniedError", + "ToolActivationError", + "ToolCheckpointRequiredError", + "ToolDeactivationError", + "ToolDescriptor", + "ToolExecutionError", + "ToolInstance", + "ToolLifecycleCache", + "ToolNotActivatedError", + "ToolRuntime", + "ToolRuntimeError", + # C5.lifecycle — schema validation + "ToolSchemaValidationError", + "validate_tool_input", + "validate_tool_output", ] diff --git a/src/cleveragents/tool/context.py b/src/cleveragents/tool/context.py new file mode 100644 index 000000000..ebc5bc290 --- /dev/null +++ b/src/cleveragents/tool/context.py @@ -0,0 +1,229 @@ +"""Tool execution context for CleverAgents v3. + +The ``ToolExecutionContext`` is provided to every tool execution, regardless of +source. It carries the plan metadata, sandbox reference, resolved resource +bindings, a change recorder, and a cancellation token so that long-running +tools can be interrupted when a plan is cancelled. + +Based on docs/specification.md Tool Interface and Architecture section. +""" + +from __future__ import annotations + +import threading +from collections import OrderedDict +from datetime import UTC, datetime +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +# --------------------------------------------------------------------------- +# Bound resource placeholder +# --------------------------------------------------------------------------- + + +class BoundResource(BaseModel): + """A resolved resource binding for a tool slot. + + At execution time each ``ResourceSlot`` declared by the tool is resolved to + a ``BoundResource`` that carries the concrete resource identity, the sandbox + path (if applicable), and the access mode. + """ + + slot_name: str = Field(..., min_length=1, description="Slot name from the tool") + resource_id: str = Field(..., min_length=1, description="Resource ULID or name") + resource_type: str = Field(..., min_length=1, description="Resource type name") + sandbox_path: str | None = Field( + default=None, + description="Sandbox-mapped path (None for non-file resources)", + ) + access: str = Field( + "read_only", description="Access mode: 'read_only' or 'read_write'" + ) + + model_config = ConfigDict(str_strip_whitespace=True) + + +# --------------------------------------------------------------------------- +# Change tracking +# --------------------------------------------------------------------------- + + +class ChangeOperation(StrEnum): + """Operation type for a recorded change.""" + + CREATE = "create" + MODIFY = "modify" + DELETE = "delete" + MOVE = "move" + + +class Change(BaseModel): + """A single recorded change produced by a tool execution. + + Every resource modification is explicit and tracked via the execution + context. The ChangeSet is built from these entries, enabling precise + rollback (replay inverse of recorded changes) and audit. + """ + + operation: ChangeOperation = Field(..., description="Type of change") + resource_id: str = Field(..., description="Resource ID affected") + path: str | None = Field(default=None, description="File path (if applicable)") + previous_content: str | None = Field( + default=None, description="Content before change" + ) + new_content: str | None = Field(default=None, description="Content after change") + metadata: dict[str, Any] = Field( + default_factory=dict, description="Additional change metadata" + ) + timestamp: str = Field( + default_factory=lambda: datetime.now(UTC).isoformat(), + description="ISO-8601 timestamp of the change", + ) + + model_config = ConfigDict(str_strip_whitespace=True) + + +# --------------------------------------------------------------------------- +# Cancellation token +# --------------------------------------------------------------------------- + + +class CancellationToken: + """Thread-safe cancellation token for tool execution. + + The plan lifecycle can set the token to signal cancellation, and tools + check ``is_cancelled`` to abort long-running operations early. + """ + + def __init__(self) -> None: + self._event = threading.Event() + + def cancel(self) -> None: + """Signal cancellation.""" + self._event.set() + + @property + def is_cancelled(self) -> bool: + """Return True if cancellation has been requested.""" + return self._event.is_set() + + def check(self) -> None: + """Raise ``ToolCancelledError`` if cancellation has been requested.""" + if self._event.is_set(): + raise ToolCancelledError("Tool execution was cancelled") + + +class ToolCancelledError(Exception): + """Raised when a tool execution is cancelled.""" + + +# --------------------------------------------------------------------------- +# Execution trace +# --------------------------------------------------------------------------- + + +class ToolExecutionTrace(BaseModel): + """Diagnostic trace for a single tool execution. + + Captures start/end timestamps, duration, result size, and any error + information for debugging and performance analysis. + """ + + tool_name: str = Field(..., description="Namespaced tool name") + started_at: str = Field(..., description="ISO-8601 start timestamp") + ended_at: str | None = Field(default=None, description="ISO-8601 end timestamp") + duration_ms: float | None = Field( + default=None, description="Duration in milliseconds" + ) + result_size_bytes: int | None = Field( + default=None, description="Size of the result payload" + ) + error: str | None = Field(default=None, description="Error message if failed") + success: bool = Field(default=True, description="Whether execution succeeded") + + model_config = ConfigDict(str_strip_whitespace=True) + + +# --------------------------------------------------------------------------- +# Tool execution context +# --------------------------------------------------------------------------- + + +class ToolExecutionContext: + """Context provided to every tool execution, regardless of source. + + Carries plan metadata, sandbox reference, resolved resource bindings, + a change list, cancellation token, and execution traces. + + Parameters + ---------- + plan_id: + The ULID of the plan that owns this execution. + plan_read_only: + Whether the plan is read-only (restricts tool access). + require_checkpoints: + Whether the plan requires all tools to be checkpointable. + sandbox_id: + Optional sandbox identifier for the execution. + resources: + Resolved resource bindings keyed by slot name. + cancellation_token: + Token for cancelling long-running tools. + metadata: + Additional plan/execution metadata. + """ + + def __init__( + self, + *, + plan_id: str, + plan_read_only: bool = False, + require_checkpoints: bool = False, + sandbox_id: str | None = None, + resources: dict[str, BoundResource] | None = None, + cancellation_token: CancellationToken | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + self.plan_id = plan_id + self.plan_read_only = plan_read_only + self.require_checkpoints = require_checkpoints + self.sandbox_id = sandbox_id + self.resources: dict[str, BoundResource] = resources or {} + self.cancellation_token = cancellation_token or CancellationToken() + self.metadata: dict[str, Any] = metadata or {} + self.changes: list[Change] = [] + self.traces: list[ToolExecutionTrace] = [] + + def record_change(self, change: Change) -> None: + """Record a change made by a tool.""" + self.changes.append(change) + + def add_trace(self, trace: ToolExecutionTrace) -> None: + """Add an execution trace entry.""" + self.traces.append(trace) + + def get_resource(self, slot_name: str) -> BoundResource: + """Get a bound resource by slot name. + + Raises ``KeyError`` if the slot is not bound. + """ + if slot_name not in self.resources: + raise KeyError( + f"No resource bound to slot '{slot_name}'. " + f"Available slots: {sorted(self.resources)}" + ) + return self.resources[slot_name] + + def as_summary(self) -> OrderedDict[str, Any]: + """Return a summary dict for diagnostics and CLI rendering.""" + result: OrderedDict[str, Any] = OrderedDict() + result["plan_id"] = self.plan_id + result["plan_read_only"] = self.plan_read_only + result["require_checkpoints"] = self.require_checkpoints + result["sandbox_id"] = self.sandbox_id + result["resource_count"] = len(self.resources) + result["change_count"] = len(self.changes) + result["trace_count"] = len(self.traces) + return result diff --git a/src/cleveragents/tool/lifecycle.py b/src/cleveragents/tool/lifecycle.py new file mode 100644 index 000000000..c3262dcb8 --- /dev/null +++ b/src/cleveragents/tool/lifecycle.py @@ -0,0 +1,649 @@ +"""Tool lifecycle runtime for CleverAgents v3. + +Implements the four-stage tool lifecycle (``discover``/``activate``/ +``execute``/``deactivate``) with: + +- ``ToolDescriptor``: lightweight metadata returned by ``discover()`` +- ``ToolInstance``: Protocol for concrete tool implementations +- ``ToolRuntime``: orchestrates lifecycle, caching, capability enforcement, + JSON Schema validation, tracing, and cancellation propagation + +The runtime maintains a per-plan activation cache so that tools are activated +at most once per plan and guaranteed to be deactivated when the plan +completes or is cancelled. + +Based on docs/specification.md Tool Interface and Architecture section. +""" + +from __future__ import annotations + +import logging +import threading +import time +from collections import OrderedDict +from datetime import UTC, datetime +from typing import Any, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field + +from cleveragents.domain.models.core.tool import Tool, ToolCapability +from cleveragents.tool.context import ( + Change, + ToolCancelledError, + ToolExecutionContext, + ToolExecutionTrace, +) +from cleveragents.tool.schema_validator import ( + ToolSchemaValidationError, + validate_tool_input, + validate_tool_output, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class ToolRuntimeError(Exception): + """Base error for tool runtime failures.""" + + +class ToolAccessDeniedError(ToolRuntimeError): + """Raised when a tool is invoked in violation of access constraints. + + For example, a tool with ``writes=True`` invoked from a read-only plan. + """ + + +class ToolCheckpointRequiredError(ToolRuntimeError): + """Raised when a plan requires checkpoints but the tool is not checkpointable.""" + + +class ToolNotActivatedError(ToolRuntimeError): + """Raised when execute is called on a tool that has not been activated.""" + + +class ToolActivationError(ToolRuntimeError): + """Raised when tool activation fails.""" + + +class ToolExecutionError(ToolRuntimeError): + """Raised when tool execution fails.""" + + +class ToolDeactivationError(ToolRuntimeError): + """Raised when tool deactivation fails.""" + + +# --------------------------------------------------------------------------- +# Tool descriptor +# --------------------------------------------------------------------------- + + +class ToolDescriptor(BaseModel): + """Lightweight metadata returned by ``discover()``. + + Describes a tool's identity, capabilities, and schema without + carrying the full implementation. + """ + + name: str = Field(..., description="Namespaced tool name") + description: str = Field(..., description="Human-readable description") + capability: ToolCapability = Field( + default_factory=ToolCapability, + description="Capability metadata", + ) + input_schema: dict[str, Any] | None = Field( + default=None, description="JSON Schema for inputs" + ) + output_schema: dict[str, Any] | None = Field( + default=None, description="JSON Schema for outputs" + ) + source: str = Field("builtin", description="Tool source identifier") + + model_config = ConfigDict(str_strip_whitespace=True) + + +# --------------------------------------------------------------------------- +# Tool result +# --------------------------------------------------------------------------- + + +class ToolResult(BaseModel): + """Structured result from a tool execution. + + Wraps the output data, any changes produced, and metadata about the + execution. + """ + + success: bool = Field(True, description="Whether execution succeeded") + data: Any = Field(default=None, description="Tool output data") + changes: list[Change] = Field(default_factory=list, description="Changes produced") + error: str | None = Field(default=None, description="Error message if failed") + + model_config = ConfigDict(str_strip_whitespace=True) + + +# --------------------------------------------------------------------------- +# Tool instance protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class ToolInstance(Protocol): + """Protocol for concrete tool implementations. + + Each tool source (MCP, Agent Skill, built-in, custom) provides an + implementation of this protocol. The lifecycle methods are called + by ``ToolRuntime`` in strict order. + """ + + def discover(self) -> ToolDescriptor: + """Return lightweight metadata about the tool. + + Called during tool registration to enumerate available tools. + Must be cheap (~50-100 tokens for Agent Skills). + """ + ... + + def activate(self, ctx: ToolExecutionContext) -> None: + """Prepare the tool for execution within a plan context. + + Called at most once per plan. May start external processes, + verify connectivity, load instructions into agent context, etc. + + Raises ``ToolActivationError`` on failure. + """ + ... + + def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult: + """Execute the tool with given parameters. + + Must only be called after ``activate()``. The ``ctx`` carries + resource bindings, sandbox state, and the cancellation token. + + Tools should check ``ctx.cancellation_token.is_cancelled`` + periodically for long-running operations. + + Returns a ``ToolResult`` with output data and any changes. + """ + ... + + def deactivate(self, ctx: ToolExecutionContext) -> None: + """Clean up after all executions within a plan are done. + + Called once per plan when the plan completes, fails, or is + cancelled. Must be idempotent and must not raise. + """ + ... + + +# --------------------------------------------------------------------------- +# Lifecycle cache +# --------------------------------------------------------------------------- + + +class _ActivationEntry: + """Internal cache entry for an activated tool instance.""" + + __slots__ = ("activated_at", "execution_count", "instance") + + def __init__(self, instance: ToolInstance, activated_at: str) -> None: + self.instance = instance + self.activated_at = activated_at + self.execution_count = 0 + + +class ToolLifecycleCache: + """Thread-safe per-plan activation cache. + + Ensures each tool is activated at most once per plan and guarantees + ``deactivate()`` is called for every activated tool when the plan + ends. + """ + + def __init__(self) -> None: + self._lock = threading.RLock() + # plan_id -> {tool_name -> _ActivationEntry} + self._cache: dict[str, dict[str, _ActivationEntry]] = {} + + def get(self, plan_id: str, tool_name: str) -> ToolInstance | None: + """Get an activated instance, or None if not cached.""" + with self._lock: + plan_cache = self._cache.get(plan_id) + if plan_cache is None: + return None + entry = plan_cache.get(tool_name) + return entry.instance if entry else None + + def put(self, plan_id: str, tool_name: str, instance: ToolInstance) -> None: + """Store an activated instance in the cache.""" + with self._lock: + if plan_id not in self._cache: + self._cache[plan_id] = {} + self._cache[plan_id][tool_name] = _ActivationEntry( + instance=instance, + activated_at=datetime.now(UTC).isoformat(), + ) + + def increment_execution(self, plan_id: str, tool_name: str) -> None: + """Increment the execution count for a cached tool.""" + with self._lock: + plan_cache = self._cache.get(plan_id) + if plan_cache and tool_name in plan_cache: + plan_cache[tool_name].execution_count += 1 + + def get_plan_tools(self, plan_id: str) -> list[str]: + """Return the names of all activated tools for a plan.""" + with self._lock: + plan_cache = self._cache.get(plan_id) + return list(plan_cache.keys()) if plan_cache else [] + + def remove(self, plan_id: str, tool_name: str) -> ToolInstance | None: + """Remove and return an activated instance from the cache.""" + with self._lock: + plan_cache = self._cache.get(plan_id) + if plan_cache and tool_name in plan_cache: + entry = plan_cache.pop(tool_name) + if not plan_cache: + del self._cache[plan_id] + return entry.instance + return None + + def remove_plan(self, plan_id: str) -> dict[str, ToolInstance]: + """Remove all activated instances for a plan. + + Returns a mapping of tool_name -> instance for deactivation. + """ + with self._lock: + plan_cache = self._cache.pop(plan_id, {}) + return {name: entry.instance for name, entry in plan_cache.items()} + + def get_stats(self, plan_id: str) -> OrderedDict[str, Any]: + """Return cache statistics for a plan.""" + with self._lock: + plan_cache = self._cache.get(plan_id, {}) + stats: OrderedDict[str, Any] = OrderedDict() + stats["plan_id"] = plan_id + stats["active_tools"] = len(plan_cache) + stats["tools"] = { + name: { + "activated_at": entry.activated_at, + "execution_count": entry.execution_count, + } + for name, entry in plan_cache.items() + } + return stats + + @property + def plan_count(self) -> int: + """Return the number of plans with active tools.""" + with self._lock: + return len(self._cache) + + +# --------------------------------------------------------------------------- +# Tool runtime +# --------------------------------------------------------------------------- + + +class ToolRuntime: + """Orchestrates the four-stage tool lifecycle. + + Responsibilities: + - Capability enforcement (read-only plans, checkpoint requirements) + - JSON Schema validation of inputs and outputs + - Per-plan activation caching with guaranteed deactivation + - Execution tracing (timestamps, duration, result size) + - Cancellation propagation to long-running tools + + Parameters + ---------- + tools: + Mapping of tool name -> Tool domain model for registered tools. + instances: + Mapping of tool name -> ToolInstance for concrete implementations. + """ + + def __init__( + self, + *, + tools: dict[str, Tool] | None = None, + instances: dict[str, ToolInstance] | None = None, + ) -> None: + self._tools: dict[str, Tool] = tools or {} + self._instances: dict[str, ToolInstance] = instances or {} + self._cache = ToolLifecycleCache() + self._lock = threading.RLock() + + # -- Registration -------------------------------------------------------- + + def register_tool(self, tool: Tool, instance: ToolInstance) -> None: + """Register a tool and its implementation. + + Parameters + ---------- + tool: + The Tool domain model describing the tool. + instance: + The concrete ToolInstance implementation. + """ + with self._lock: + self._tools[tool.name] = tool + self._instances[tool.name] = instance + + def unregister_tool(self, tool_name: str) -> None: + """Remove a tool registration. + + Does not deactivate existing plan-level activations; those are + handled by ``deactivate_plan()``. + """ + with self._lock: + self._tools.pop(tool_name, None) + self._instances.pop(tool_name, None) + + def get_tool(self, tool_name: str) -> Tool | None: + """Get a registered Tool domain model by name.""" + return self._tools.get(tool_name) + + def list_tools(self) -> list[str]: + """Return names of all registered tools.""" + return sorted(self._tools.keys()) + + # -- Discovery ----------------------------------------------------------- + + def discover(self, tool_name: str) -> ToolDescriptor: + """Invoke the ``discover()`` lifecycle hook. + + Returns a ``ToolDescriptor`` with the tool's metadata. + """ + instance = self._get_instance(tool_name) + return instance.discover() + + # -- Activation ---------------------------------------------------------- + + def activate(self, tool_name: str, ctx: ToolExecutionContext) -> None: + """Activate a tool for a plan context. + + Uses the lifecycle cache to ensure each tool is activated at most + once per plan. Performs capability pre-checks before activation. + + Raises + ------ + ToolAccessDeniedError: + If the plan is read-only and the tool writes. + ToolCheckpointRequiredError: + If the plan requires checkpoints and the tool is not checkpointable. + ToolActivationError: + If the tool's ``activate()`` hook fails. + """ + tool = self._get_tool(tool_name) + self._enforce_capabilities(tool, ctx) + + # Check cache - already activated for this plan? + cached = self._cache.get(ctx.plan_id, tool_name) + if cached is not None: + return + + instance = self._get_instance(tool_name) + try: + instance.activate(ctx) + except ToolCancelledError: + raise + except Exception as exc: + raise ToolActivationError( + f"Failed to activate tool '{tool_name}': {exc}" + ) from exc + + self._cache.put(ctx.plan_id, tool_name, instance) + logger.info( + "Tool activated", + extra={"tool": tool_name, "plan_id": ctx.plan_id}, + ) + + # -- Execution ----------------------------------------------------------- + + def execute( + self, + tool_name: str, + params: dict[str, Any], + ctx: ToolExecutionContext, + ) -> ToolResult: + """Execute a tool with given parameters. + + Performs the full execution pipeline: + 1. Check cancellation + 2. Auto-activate if not yet activated for this plan + 3. Enforce capability flags + 4. Validate inputs against JSON Schema + 5. Execute the tool + 6. Validate outputs against JSON Schema + 7. Record trace + + Parameters + ---------- + tool_name: + Namespaced name of the tool to execute. + params: + Input parameters for the tool. + ctx: + Execution context with plan metadata, resources, cancellation. + + Returns + ------- + ToolResult: + The structured result from the tool execution. + """ + # 1. Check cancellation before starting + ctx.cancellation_token.check() + + tool = self._get_tool(tool_name) + + # 2. Enforce capability flags + self._enforce_capabilities(tool, ctx) + + # 3. Auto-activate if needed + self.activate(tool_name, ctx) + + # 4. Validate inputs + if tool.input_schema: + try: + validate_tool_input(params, tool.input_schema) + except ToolSchemaValidationError as exc: + return ToolResult( + success=False, + error=f"Input validation failed: {exc}", + ) + + # 5. Execute with tracing + started_at = datetime.now(UTC) + trace = ToolExecutionTrace( + tool_name=tool_name, + started_at=started_at.isoformat(), + ) + + try: + # Check cancellation again right before execution + ctx.cancellation_token.check() + + instance = self._cache.get(ctx.plan_id, tool_name) + if instance is None: + raise ToolNotActivatedError( + f"Tool '{tool_name}' is not activated for plan '{ctx.plan_id}'" + ) + + start_time = time.monotonic() + result = instance.execute(params, ctx) + elapsed_ms = (time.monotonic() - start_time) * 1000 + + # Record trace + ended_at = datetime.now(UTC) + trace.ended_at = ended_at.isoformat() + trace.duration_ms = round(elapsed_ms, 2) + trace.success = result.success + + if result.data is not None: + trace.result_size_bytes = len(str(result.data).encode()) + + if result.error: + trace.error = result.error + + self._cache.increment_execution(ctx.plan_id, tool_name) + + except ToolCancelledError: + ended_at = datetime.now(UTC) + trace.ended_at = ended_at.isoformat() + trace.duration_ms = round((ended_at - started_at).total_seconds() * 1000, 2) + trace.success = False + trace.error = "Cancelled" + ctx.add_trace(trace) + raise + + except Exception as exc: + ended_at = datetime.now(UTC) + trace.ended_at = ended_at.isoformat() + trace.duration_ms = round((ended_at - started_at).total_seconds() * 1000, 2) + trace.success = False + trace.error = str(exc) + ctx.add_trace(trace) + raise ToolExecutionError( + f"Tool '{tool_name}' execution failed: {exc}" + ) from exc + + ctx.add_trace(trace) + + # 6. Validate outputs + if tool.output_schema and result.data is not None: + try: + validate_tool_output(result.data, tool.output_schema) + except ToolSchemaValidationError as exc: + logger.warning( + "Tool output validation failed", + extra={"tool": tool_name, "error": str(exc)}, + ) + return ToolResult( + success=False, + data=result.data, + changes=result.changes, + error=f"Output validation failed: {exc}", + ) + + # Record changes from result into context + for change in result.changes: + ctx.record_change(change) + + return result + + # -- Deactivation -------------------------------------------------------- + + def deactivate(self, tool_name: str, ctx: ToolExecutionContext) -> None: + """Deactivate a single tool for a plan. + + Removes the tool from the activation cache and calls the tool's + ``deactivate()`` hook. Idempotent -- does nothing if the tool + is not activated for the plan. + """ + instance = self._cache.remove(ctx.plan_id, tool_name) + if instance is None: + return + + try: + instance.deactivate(ctx) + except Exception as exc: + logger.warning( + "Tool deactivation failed (non-fatal)", + extra={"tool": tool_name, "plan_id": ctx.plan_id, "error": str(exc)}, + ) + + logger.info( + "Tool deactivated", + extra={"tool": tool_name, "plan_id": ctx.plan_id}, + ) + + def deactivate_plan(self, ctx: ToolExecutionContext) -> None: + """Deactivate all tools for a plan. + + Called when a plan completes, fails, or is cancelled. Guarantees + every activated tool receives a ``deactivate()`` call. + """ + instances = self._cache.remove_plan(ctx.plan_id) + for tool_name, instance in instances.items(): + try: + instance.deactivate(ctx) + except Exception as exc: + logger.warning( + "Tool deactivation failed during plan cleanup (non-fatal)", + extra={ + "tool": tool_name, + "plan_id": ctx.plan_id, + "error": str(exc), + }, + ) + + if instances: + logger.info( + "All plan tools deactivated", + extra={ + "plan_id": ctx.plan_id, + "tool_count": len(instances), + }, + ) + + # -- Introspection ------------------------------------------------------- + + def get_cache_stats(self, plan_id: str) -> OrderedDict[str, Any]: + """Return activation cache statistics for a plan.""" + return self._cache.get_stats(plan_id) + + @property + def cache_plan_count(self) -> int: + """Return the number of plans with active tool caches.""" + return self._cache.plan_count + + # -- Internal helpers ---------------------------------------------------- + + def _get_tool(self, tool_name: str) -> Tool: + """Get a registered Tool or raise.""" + tool = self._tools.get(tool_name) + if tool is None: + raise ToolRuntimeError(f"Tool '{tool_name}' is not registered") + return tool + + def _get_instance(self, tool_name: str) -> ToolInstance: + """Get a registered ToolInstance or raise.""" + instance = self._instances.get(tool_name) + if instance is None: + raise ToolRuntimeError( + f"No implementation registered for tool '{tool_name}'" + ) + return instance + + @staticmethod + def _enforce_capabilities(tool: Tool, ctx: ToolExecutionContext) -> None: + """Enforce capability constraints against the execution context. + + Raises + ------ + ToolAccessDeniedError: + If the plan is read-only and the tool has ``writes=True``. + ToolCheckpointRequiredError: + If the plan requires checkpoints and the tool is not + ``checkpointable``. + """ + cap = tool.capability + + # Read-only plan cannot use tools that write + if ctx.plan_read_only and not cap.read_only and cap.writes: + raise ToolAccessDeniedError( + f"Tool '{tool.name}' has writes=True but plan " + f"'{ctx.plan_id}' is read-only" + ) + + # Checkpoint-required plan cannot use non-checkpointable tools + if ctx.require_checkpoints and not cap.checkpointable: + raise ToolCheckpointRequiredError( + f"Tool '{tool.name}' is not checkpointable but plan " + f"'{ctx.plan_id}' requires checkpoints" + ) diff --git a/src/cleveragents/tool/schema_validator.py b/src/cleveragents/tool/schema_validator.py new file mode 100644 index 000000000..b78a9f94a --- /dev/null +++ b/src/cleveragents/tool/schema_validator.py @@ -0,0 +1,124 @@ +"""JSON Schema validation for tool inputs and outputs. + +Validates tool parameters against their declared ``input_schema`` and +validates tool results against their declared ``output_schema`` using +the ``jsonschema`` library (JSON Schema draft 2020-12). + +Based on docs/specification.md Tool Execution Flow section. +""" + +from __future__ import annotations + +from typing import Any + +import jsonschema +import jsonschema.validators + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class ToolSchemaValidationError(Exception): + """Raised when tool input or output fails JSON Schema validation. + + Attributes + ---------- + errors: + List of individual validation error messages. + schema: + The JSON Schema that was used for validation. + instance: + The value that failed validation. + """ + + def __init__( + self, + message: str, + *, + errors: list[str] | None = None, + schema: dict[str, Any] | None = None, + instance: Any = None, + ) -> None: + super().__init__(message) + self.errors = errors or [] + self.schema = schema + self.instance = instance + + +# --------------------------------------------------------------------------- +# Validation functions +# --------------------------------------------------------------------------- + + +def validate_tool_input( + params: dict[str, Any], + schema: dict[str, Any], +) -> None: + """Validate tool input parameters against a JSON Schema. + + Parameters + ---------- + params: + The parameters to validate. + schema: + The JSON Schema to validate against. + + Raises + ------ + ToolSchemaValidationError: + If validation fails, with details about each error. + """ + _validate(params, schema, context="input") + + +def validate_tool_output( + result: Any, + schema: dict[str, Any], +) -> None: + """Validate tool output against a JSON Schema. + + Parameters + ---------- + result: + The output to validate. + schema: + The JSON Schema to validate against. + + Raises + ------ + ToolSchemaValidationError: + If validation fails, with details about each error. + """ + _validate(result, schema, context="output") + + +def _validate( + instance: Any, + schema: dict[str, Any], + *, + context: str, +) -> None: + """Internal validation helper. + + Uses ``jsonschema.Draft202012Validator`` for JSON Schema draft 2020-12 + compliance (per spec). + """ + validator_cls = jsonschema.Draft202012Validator + validator = validator_cls(schema) + errors = sorted(validator.iter_errors(instance), key=lambda e: list(e.path)) + + if errors: + messages = [] + for err in errors: + path = ".".join(str(p) for p in err.absolute_path) if err.path else "" + messages.append(f" [{path}] {err.message}") + + raise ToolSchemaValidationError( + f"Tool {context} schema validation failed " + f"({len(errors)} error{'s' if len(errors) > 1 else ''}):\n" + + "\n".join(messages), + errors=messages, + schema=schema, + instance=instance, + )