cleveragents/cleveractors-core
0
0
Fork 0
Code Issues 17 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(agents): surface timeout budget in tool schemas and errors #84

Merged
CoreRasurae merged 3 commits from feature/m2-tool-timeout-awareness into master 2026-07-31 08:31:16 +00:00
Conversation 2 Commits 3 Files Changed 9
+149 -31
9 changed files with 149 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.forgejo/workflows/ci.yml → .gitea/workflows/ci.yml
+98 -18
View File
@@ -8,23 +8,28 @@ on:
env:
UV_VERSION: "0.8.0"
PYTHON_VERSION: "3.11"
PYTHON_VERSION: "3.13"
NOX_DEFAULT_VENV_BACKEND: "uv"
FORGEJO_INTERNAL_URL: "http://forgejo-gitea-http.forgejo.svc.cluster.local:3000"
jobs:
lint:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -55,16 +60,20 @@ jobs:
typecheck:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -91,16 +100,20 @@ jobs:
security:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -131,16 +144,20 @@ jobs:
quality:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -167,16 +184,20 @@ jobs:
unit_tests:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install system dependencies (nodejs for checkout, git for tests)
run: |
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -203,16 +224,20 @@ jobs:
integration_tests:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install system dependencies (nodejs for checkout, git for tests)
run: |
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -239,17 +264,21 @@ jobs:
coverage:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
needs: [lint, typecheck, security, quality, unit_tests]
steps:
- name: Install system dependencies (nodejs for checkout, git for tests)
run: |
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -288,16 +317,20 @@ jobs:
build:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
@@ -319,6 +352,53 @@ jobs:
path: dist/*.whl
retention-days: 30
benchmark:
runs-on: docker
container:
image: python:3.13-slim
continue-on-error: true
steps:
- name: Install Node.js (required by actions/checkout)
run: |
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Ensure origin/master is available for comparison
run: |
git fetch origin master:refs/remotes/origin/master
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Cache uv packages
uses: actions/cache@v3
with:
path: ~/.cache/uv
key: uv-${{ hashFiles('pyproject.toml') }}
restore-keys: |
uv-
- name: Run ASV benchmark regression check via nox
run: |
mkdir -p build
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-output.log
- name: Upload benchmark log artifact
if: always()
uses: actions/upload-artifact@v3
with:
name: ci-logs-benchmark
path: build/nox-benchmark-output.log
retention-days: 30
status-check:
if: always()
needs:
@@ -332,7 +412,7 @@ jobs:
- build
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Check required job results
run: |
.forgejo/workflows/release.yml → .gitea/workflows/release.yml
+13 -5
View File
@@ -7,23 +7,28 @@ on:
env:
UV_VERSION: "0.8.0"
PYTHON_VERSION: "3.11"
PYTHON_VERSION: "3.13"
NOX_DEFAULT_VENV_BACKEND: "uv"
FORGEJO_INTERNAL_URL: "http://forgejo-gitea-http.forgejo.svc.cluster.local:3000"
jobs:
build-wheel:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
steps:
- name: Install Node.js (required by actions/checkout)
run: |
apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
- name: Install uv and nox
run: |
git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/"
pip install -q "uv==${UV_VERSION}" nox
- name: Build wheel via nox
@@ -84,12 +89,15 @@ jobs:
create-release:
runs-on: docker
container:
image: python:3.11-slim
image: python:3.13-slim
needs: [build-wheel, build-docker]
steps:
- name: Install system dependencies
run: |
apt-get update && apt-get install -y -qq nodejs curl jq git && rm -rf /var/lib/apt/lists/*
sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true
apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
apt-get install -y -qq nodejs curl jq git
rm -rf /var/lib/apt/lists/*
- uses: actions/checkout@v4
with:
CHANGELOG.md
+1
View File
@@ -168,6 +168,7 @@ and this project adheres to [Clever Semantic Versioning](https://www.w3.org/subm
### Changed
- **Timeout Awareness Surfaced to the LLM for `shell` and `http_request` Tools (issue #82)** (`agents/llm_tools.py`, `agents/tool.py`): The LLM-facing tool-calling schemas for `shell` and `http_request` now state that execution is bounded by the agent's configured `timeout` (seconds) — previously only `file_read`'s `max_chars` guidance was documented in the schema, leaving the model unaware that these two tools could be killed/aborted mid-call. When a timeout is actually triggered, `ToolAgent._execute_shell_command` and `ToolAgent._http_request_tool` now name `timeout` as the adjustable agent config field in the raised `ExecutionError`, in addition to the elapsed seconds already reported. `_http_request_tool` also now catches the timeout condition distinctly from other request failures (connection errors, invalid URLs, etc.), which previously all collapsed into one generic "HTTP request failed" message with no indication a timeout occurred. No change to the underlying `timeout` enforcement mechanics (§4.5) — this is schema/message content only. BDD: scenarios in `features/tool_agent.feature` and `features/llm_tools_coverage.feature`.
- **Real LangChain `usage_metadata` token extraction**: `LLMAgent.process_message()` now reads token counts directly from LangChain response metadata instead of estimating via tiktoken or a 4-chars/token heuristic. `ActorResult.prompt_tokens`/`completion_tokens` now reflect actual LLM billing data.
- **`ActorResult`/`NodeUsage` moved to canonical `cleveractors.result` module**: Both types are now defined in `src/cleveractors/result.py` (ADR-2027 §3.1) and re-exported from `cleveractors.runtime` for backward compatibility. `cleveractors.__init__` updated to import from `result` instead of `runtime`.
- **RxPy Architecture**: Converted routing system from synchronous to reactive (RxPy) streams using Observable/Subject patterns for asynchronous message flow.
features/llm_tools_coverage.feature
+13 -1
View File
@@ -31,4 +31,16 @@ Feature: LLM Tools Coverage
Scenario: normalize_tool_entry rejects non-str non-dict type
Given an llm_tools coverage test environment
When I call normalize_tool_entry with integer 42
Then a ConfigurationError mentioning Invalid tool config entry is raised
Then a ConfigurationError mentioning Invalid tool config entry is raised
Scenario: shell tool schema description mentions the configurable timeout
Given an llm_tools coverage test environment
When I call normalize_tool_entry with string "shell"
Then the result description should mention "timeout"
And the result description should mention "config"
Scenario: http_request tool schema description mentions the configurable timeout
Given an llm_tools coverage test environment
When I call normalize_tool_entry with string "http_request"
Then the result description should mention "timeout"
And the result description should mention "config"
features/steps/llm_tools_coverage_steps.py
+7
View File
@@ -73,3 +73,10 @@ def step_lt_same_dict(context):
def step_lt_config_error(context, text):
assert context.error is not None
assert text in context.error
@then('the result description should mention "{text}"')
def step_lt_description_mentions(context, text):
assert context.result is not None
description = context.result["function"]["description"]
assert text in description, f"Expected '{text}' in description: {description!r}"
features/tool_agent.feature
+3 -1
View File
@@ -242,6 +242,7 @@ Feature: ToolAgent Functionality
When I create the ToolAgent
And I process a message with the ToolAgent: "sleep 1"
Then tool execution should fail with message containing "timed out after 0.1 seconds"
And tool execution should fail with message containing "'timeout' config value"
Scenario: Tool agent shell command with non-zero exit code
Given I am running in unsafe mode
@@ -536,7 +537,8 @@ Feature: ToolAgent Functionality
"""
{"tool": "http_request", "args": {"url": "https://httpbin.org/delay/1", "method": "GET"}}
"""
Then tool execution should fail with message containing "HTTP request failed"
Then tool execution should fail with message containing "timed out after 0.001 seconds"
And tool execution should fail with message containing "'timeout' config value"
Scenario: Tool agent http_request tool with invalid URL
Given a ToolAgent is configured with name "http_invalid_agent" and config
robot/version.robot
+3 -3
View File
@@ -35,7 +35,7 @@ Core Modules Are Importable
[Documentation] Verify core modules import correctly
${result}= Run Process python -c
... from cleveractors.core.config import ConfigurationManager, SchemaValidator\; from cleveractors.core.exceptions import ConfigurationError, AgentCreationError, RoutingError, TemplateError, ExecutionError, ApplicationError, StreamRoutingError\; from cleveractors.core.progress import ProgressBarManager\; print("OK")
... timeout=5s
... timeout=30s
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
@@ -43,7 +43,7 @@ Templates Modules Are Importable
[Documentation] Verify template modules import correctly
${result}= Run Process python -c
... from cleveractors.templates.renderer import TemplateEngine, TemplateRenderer\; from cleveractors.templates.registry import TemplateRegistry\; from cleveractors.templates.enhanced_registry import EnhancedTemplateRegistry\; print("OK")
... timeout=5s
... timeout=30s
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
@@ -51,6 +51,6 @@ LangGraph Modules Are Importable
[Documentation] Verify LangGraph modules import correctly
${result}= Run Process python -c
... from cleveractors.langgraph.graph import LangGraph\; from cleveractors.langgraph.bridge import RxPyLangGraphBridge\; from cleveractors.langgraph.state import GraphState\; from cleveractors.langgraph.nodes import NodeConfig, NodeType\; print("OK")
... timeout=5s
... timeout=30s
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} OK
src/cleveractors/agents/llm_tools.py
+2 -2
View File
@@ -70,7 +70,7 @@ _BUILTIN_TOOL_SCHEMAS: dict[str, dict[str, Any]] = {
},
},
"http_request": {
"description": "Make an HTTP request and return the response.",
"description": "Make an HTTP request and return the response. Execution is bounded by the agent's configured `timeout` (seconds, default 1); requests that exceed it are aborted and return a timeout error — increase `timeout` in the tool agent config for slow endpoints.",
"parameters": {
"type": "object",
"properties": {
@@ -208,7 +208,7 @@ _BUILTIN_TOOL_SCHEMAS: dict[str, dict[str, Any]] = {
},
},
"shell": {
"description": "Execute a shell command and return stdout. Requires allow_shell. Use for: find, grep, cat. Prefer file_read for reading files (has max_chars to prevent context overflow). When creating files in new directories, always use 'mkdir -p <dir> && touch <path>'.",
"description": "Execute a shell command and return stdout. Requires allow_shell. Use for: find, grep, cat. Prefer file_read for reading files (has max_chars to prevent context overflow). When creating files in new directories, always use 'mkdir -p <dir> && touch <path>'. Execution is bounded by the agent's configured `timeout` (seconds, default 1); commands that exceed it are killed and return a timeout error — increase `timeout` in the tool agent config for long-running commands.",
"parameters": {
"type": "object",
"properties": {
src/cleveractors/agents/tool.py
+9 -1
View File
@@ -351,7 +351,9 @@ class ToolAgent(Agent):
except asyncio.TimeoutError as timeout_err:
raise ExecutionError(
f"Command '{command}' timed out after {self.timeout} seconds"
f"Command '{command}' timed out after {self.timeout} seconds. "
"Increase the tool agent's 'timeout' config value (in seconds) "
"if this command legitimately needs more time."
) from timeout_err
async def _shell_tool(
@@ -603,6 +605,12 @@ class ToolAgent(Agent):
) as response:
content = await response.text()
return f"Status: {response.status}\n\n{content}"
except asyncio.TimeoutError as timeout_err:
raise ExecutionError(
f"HTTP request to '{url}' timed out after {self.timeout} seconds. "
"Increase the tool agent's 'timeout' config value (in seconds) "
"if this endpoint legitimately needs more time."
) from timeout_err
except Exception as e:
raise ExecutionError(f"HTTP request failed: {e}") from e