237 lines
8.6 KiB
Python
237 lines
8.6 KiB
Python
"""Step definitions for inline tool executor tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.domain.models.core.skill import SkillInlineTool
|
|
from cleveragents.domain.models.core.tool import ToolCapability, ToolSource
|
|
from cleveragents.skills.context import SkillContext
|
|
from cleveragents.skills.inline_executor import InlineToolExecutor
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Givens — Tool construction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('an inline tool with code that prints "{message}"')
|
|
def inline_tool_with_print(context: Context, message: str) -> None:
|
|
"""Create an inline tool that prints a message."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Print message tool",
|
|
source=ToolSource.CUSTOM,
|
|
code=f'print("{message}")',
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given("an inline tool with code that sleeps for 5 seconds")
|
|
def inline_tool_with_sleep(context: Context) -> None:
|
|
"""Create an inline tool that sleeps."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Sleeping tool",
|
|
source=ToolSource.CUSTOM,
|
|
code="import time; time.sleep(5)",
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given("an inline tool with code that prints 2000 bytes of output")
|
|
def inline_tool_with_large_output(context: Context) -> None:
|
|
"""Create an inline tool that produces large output."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Large output tool",
|
|
source=ToolSource.CUSTOM,
|
|
code='print("X" * 2000)',
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given('an inline tool with writes capability and code that prints "{message}"')
|
|
def inline_tool_with_writes(context: Context, message: str) -> None:
|
|
"""Create an inline tool with writes capability."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Write-capable tool",
|
|
source=ToolSource.CUSTOM,
|
|
code=f'print("{message}")',
|
|
capability=ToolCapability(writes=True),
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given("an inline tool with no code")
|
|
def inline_tool_no_code(context: Context) -> None:
|
|
"""Create an inline tool without code."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="No-code tool",
|
|
source=ToolSource.CUSTOM,
|
|
code=None,
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given("an inline tool with unsupported source type")
|
|
def inline_tool_unsupported_source(context: Context) -> None:
|
|
"""Create an inline tool with unsupported source."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Unsupported source tool",
|
|
source=ToolSource.BUILTIN,
|
|
code="print('hello')",
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
@given("an inline tool with code that raises an exception")
|
|
def inline_tool_with_error(context: Context) -> None:
|
|
"""Create an inline tool that raises an exception."""
|
|
context.inline_tool = SkillInlineTool(
|
|
description="Error tool",
|
|
source=ToolSource.CUSTOM,
|
|
code='raise RuntimeError("intentional error")',
|
|
)
|
|
context.inline_executor = InlineToolExecutor()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Givens — Context construction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a writable skill context with sandbox "{sandbox}"')
|
|
def writable_skill_context(context: Context, sandbox: str) -> None:
|
|
"""Create a writable SkillContext."""
|
|
context.inline_context = SkillContext(
|
|
plan_id="plan-inline",
|
|
project_id="proj-inline",
|
|
sandbox_path=Path(sandbox),
|
|
read_only=False,
|
|
)
|
|
|
|
|
|
@given('a read-only skill context with sandbox "{sandbox}"')
|
|
def readonly_skill_context(context: Context, sandbox: str) -> None:
|
|
"""Create a read-only SkillContext."""
|
|
context.inline_context = SkillContext(
|
|
plan_id="plan-inline-ro",
|
|
project_id="proj-inline-ro",
|
|
sandbox_path=Path(sandbox),
|
|
read_only=True,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Givens — Executor configuration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("an inline executor with max runtime {seconds:g} seconds")
|
|
def inline_executor_with_timeout(context: Context, seconds: float) -> None:
|
|
"""Create an executor with custom timeout."""
|
|
context.inline_executor = InlineToolExecutor(max_runtime_seconds=seconds)
|
|
|
|
|
|
@given("an inline executor with max output {nbytes:d} bytes")
|
|
def inline_executor_with_max_output(context: Context, nbytes: int) -> None:
|
|
"""Create an executor with custom output limit."""
|
|
context.inline_executor = InlineToolExecutor(max_output_bytes=nbytes)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When — Execution
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I execute the inline tool")
|
|
def execute_inline_tool(context: Context) -> None:
|
|
"""Execute the inline tool with the configured executor and context."""
|
|
context.inline_result = context.inline_executor.execute(
|
|
tool=context.inline_tool,
|
|
context=context.inline_context,
|
|
input_data={},
|
|
)
|
|
|
|
|
|
@when("I validate the inline tool")
|
|
def validate_inline_tool(context: Context) -> None:
|
|
"""Validate the inline tool."""
|
|
context.inline_validation_errors = context.inline_executor.validate_tool(
|
|
context.inline_tool
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then — Result assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the inline result should be successful")
|
|
def inline_result_successful(context: Context) -> None:
|
|
"""Assert the inline result is successful."""
|
|
assert context.inline_result.success is True, (
|
|
f"Expected success but got error: {context.inline_result.error_message}"
|
|
)
|
|
|
|
|
|
@then("the inline result should not be successful")
|
|
def inline_result_not_successful(context: Context) -> None:
|
|
"""Assert the inline result is not successful."""
|
|
assert context.inline_result.success is False, "Expected failure but got success"
|
|
|
|
|
|
@then('the inline result output should contain "{expected}"')
|
|
def inline_result_output_contains(context: Context, expected: str) -> None:
|
|
"""Assert the inline result output contains expected text."""
|
|
output = str(context.inline_result.output or "")
|
|
assert expected in output, f"Expected '{expected}' in output '{output}'"
|
|
|
|
|
|
@then('the inline result error message should contain "{expected}"')
|
|
def inline_result_error_contains(context: Context, expected: str) -> None:
|
|
"""Assert the inline result error contains expected text."""
|
|
msg = context.inline_result.error_message or ""
|
|
assert expected in msg, f"Expected '{expected}' in error '{msg}'"
|
|
|
|
|
|
@then("the inline result should be truncated")
|
|
def inline_result_truncated(context: Context) -> None:
|
|
"""Assert the inline result output was truncated."""
|
|
assert context.inline_result.truncated is True, "Expected truncated=True"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then — Validation assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('the inline validation errors should contain "{expected}"')
|
|
def inline_validation_errors_contain(context: Context, expected: str) -> None:
|
|
"""Assert validation errors contain expected text."""
|
|
errors = context.inline_validation_errors
|
|
assert any(expected in e for e in errors), (
|
|
f"Expected error containing '{expected}' in {errors}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then — Change tracker assertions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the skill context change tracker should have {count:d} inline records")
|
|
def inline_tracker_count(context: Context, count: int) -> None:
|
|
"""Assert change tracker record count."""
|
|
actual = len(context.inline_context.change_tracker)
|
|
assert actual == count, f"Expected {count} records, got {actual}"
|
|
|
|
|
|
@then('the last inline tracker record tool_name should be "{expected}"')
|
|
def inline_tracker_last_tool(context: Context, expected: str) -> None:
|
|
"""Assert last tracker record tool_name."""
|
|
record = context.inline_context.change_tracker[-1]
|
|
assert record["tool_name"] == expected, (
|
|
f"Expected tool_name '{expected}', got '{record['tool_name']}'"
|
|
)
|