diff --git a/.forgejo/workflows/ci.yml b/.gitea/workflows/ci.yml similarity index 64% rename from .forgejo/workflows/ci.yml rename to .gitea/workflows/ci.yml index 4effe3c..56555b6 100644 --- a/.forgejo/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -8,23 +8,28 @@ on: env: UV_VERSION: "0.8.0" - PYTHON_VERSION: "3.11" + PYTHON_VERSION: "3.13" NOX_DEFAULT_VENV_BACKEND: "uv" + FORGEJO_INTERNAL_URL: "http://forgejo-gitea-http.forgejo.svc.cluster.local:3000" jobs: lint: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -55,16 +60,20 @@ jobs: typecheck: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -91,16 +100,20 @@ jobs: security: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -131,16 +144,20 @@ jobs: quality: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -167,16 +184,20 @@ jobs: unit_tests: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install system dependencies (nodejs for checkout, git for tests) run: | - apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -203,16 +224,20 @@ jobs: integration_tests: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install system dependencies (nodejs for checkout, git for tests) run: | - apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -239,17 +264,21 @@ jobs: coverage: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim needs: [lint, typecheck, security, quality, unit_tests] steps: - name: Install system dependencies (nodejs for checkout, git for tests) run: | - apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -288,16 +317,20 @@ jobs: build: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Cache uv packages @@ -319,6 +352,53 @@ jobs: path: dist/*.whl retention-days: 30 + benchmark: + runs-on: docker + container: + image: python:3.13-slim + continue-on-error: true + steps: + - name: Install Node.js (required by actions/checkout) + run: | + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Ensure origin/master is available for comparison + run: | + git fetch origin master:refs/remotes/origin/master + + - name: Install uv and nox + run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" + pip install -q "uv==${UV_VERSION}" nox + + - name: Cache uv packages + uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: uv-${{ hashFiles('pyproject.toml') }} + restore-keys: | + uv- + + - name: Run ASV benchmark regression check via nox + run: | + mkdir -p build + nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-output.log + + - name: Upload benchmark log artifact + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-logs-benchmark + path: build/nox-benchmark-output.log + retention-days: 30 + status-check: if: always() needs: @@ -332,7 +412,7 @@ jobs: - build runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Check required job results run: | diff --git a/.forgejo/workflows/release.yml b/.gitea/workflows/release.yml similarity index 83% rename from .forgejo/workflows/release.yml rename to .gitea/workflows/release.yml index 22d8aee..8b5c27d 100644 --- a/.forgejo/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -7,23 +7,28 @@ on: env: UV_VERSION: "0.8.0" - PYTHON_VERSION: "3.11" + PYTHON_VERSION: "3.13" NOX_DEFAULT_VENV_BACKEND: "uv" + FORGEJO_INTERNAL_URL: "http://forgejo-gitea-http.forgejo.svc.cluster.local:3000" jobs: build-wheel: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim steps: - name: Install Node.js (required by actions/checkout) run: | - apt-get update && apt-get install -y -qq nodejs && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 - name: Install uv and nox run: | + git config --global url."${FORGEJO_INTERNAL_URL%/}/".insteadOf "https://git.cleverthis.com/" pip install -q "uv==${UV_VERSION}" nox - name: Build wheel via nox @@ -84,12 +89,15 @@ jobs: create-release: runs-on: docker container: - image: python:3.11-slim + image: python:3.13-slim needs: [build-wheel, build-docker] steps: - name: Install system dependencies run: | - apt-get update && apt-get install -y -qq nodejs curl jq git && rm -rf /var/lib/apt/lists/* + sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources 2>/dev/null || true + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update + apt-get install -y -qq nodejs curl jq git + rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e8dea5e..5686e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -168,6 +168,7 @@ and this project adheres to [Clever Semantic Versioning](https://www.w3.org/subm ### Changed +- **Timeout Awareness Surfaced to the LLM for `shell` and `http_request` Tools (issue #82)** (`agents/llm_tools.py`, `agents/tool.py`): The LLM-facing tool-calling schemas for `shell` and `http_request` now state that execution is bounded by the agent's configured `timeout` (seconds) — previously only `file_read`'s `max_chars` guidance was documented in the schema, leaving the model unaware that these two tools could be killed/aborted mid-call. When a timeout is actually triggered, `ToolAgent._execute_shell_command` and `ToolAgent._http_request_tool` now name `timeout` as the adjustable agent config field in the raised `ExecutionError`, in addition to the elapsed seconds already reported. `_http_request_tool` also now catches the timeout condition distinctly from other request failures (connection errors, invalid URLs, etc.), which previously all collapsed into one generic "HTTP request failed" message with no indication a timeout occurred. No change to the underlying `timeout` enforcement mechanics (§4.5) — this is schema/message content only. BDD: scenarios in `features/tool_agent.feature` and `features/llm_tools_coverage.feature`. - **Real LangChain `usage_metadata` token extraction**: `LLMAgent.process_message()` now reads token counts directly from LangChain response metadata instead of estimating via tiktoken or a 4-chars/token heuristic. `ActorResult.prompt_tokens`/`completion_tokens` now reflect actual LLM billing data. - **`ActorResult`/`NodeUsage` moved to canonical `cleveractors.result` module**: Both types are now defined in `src/cleveractors/result.py` (ADR-2027 §3.1) and re-exported from `cleveractors.runtime` for backward compatibility. `cleveractors.__init__` updated to import from `result` instead of `runtime`. - **RxPy Architecture**: Converted routing system from synchronous to reactive (RxPy) streams using Observable/Subject patterns for asynchronous message flow. diff --git a/features/llm_tools_coverage.feature b/features/llm_tools_coverage.feature index 51e4c97..dde448a 100644 --- a/features/llm_tools_coverage.feature +++ b/features/llm_tools_coverage.feature @@ -31,4 +31,16 @@ Feature: LLM Tools Coverage Scenario: normalize_tool_entry rejects non-str non-dict type Given an llm_tools coverage test environment When I call normalize_tool_entry with integer 42 - Then a ConfigurationError mentioning Invalid tool config entry is raised \ No newline at end of file + Then a ConfigurationError mentioning Invalid tool config entry is raised + + Scenario: shell tool schema description mentions the configurable timeout + Given an llm_tools coverage test environment + When I call normalize_tool_entry with string "shell" + Then the result description should mention "timeout" + And the result description should mention "config" + + Scenario: http_request tool schema description mentions the configurable timeout + Given an llm_tools coverage test environment + When I call normalize_tool_entry with string "http_request" + Then the result description should mention "timeout" + And the result description should mention "config" \ No newline at end of file diff --git a/features/steps/llm_tools_coverage_steps.py b/features/steps/llm_tools_coverage_steps.py index f58aba6..64224aa 100644 --- a/features/steps/llm_tools_coverage_steps.py +++ b/features/steps/llm_tools_coverage_steps.py @@ -73,3 +73,10 @@ def step_lt_same_dict(context): def step_lt_config_error(context, text): assert context.error is not None assert text in context.error + + +@then('the result description should mention "{text}"') +def step_lt_description_mentions(context, text): + assert context.result is not None + description = context.result["function"]["description"] + assert text in description, f"Expected '{text}' in description: {description!r}" diff --git a/features/tool_agent.feature b/features/tool_agent.feature index ef86c9a..64d3c08 100644 --- a/features/tool_agent.feature +++ b/features/tool_agent.feature @@ -242,6 +242,7 @@ Feature: ToolAgent Functionality When I create the ToolAgent And I process a message with the ToolAgent: "sleep 1" Then tool execution should fail with message containing "timed out after 0.1 seconds" + And tool execution should fail with message containing "'timeout' config value" Scenario: Tool agent shell command with non-zero exit code Given I am running in unsafe mode @@ -536,7 +537,8 @@ Feature: ToolAgent Functionality """ {"tool": "http_request", "args": {"url": "https://httpbin.org/delay/1", "method": "GET"}} """ - Then tool execution should fail with message containing "HTTP request failed" + Then tool execution should fail with message containing "timed out after 0.001 seconds" + And tool execution should fail with message containing "'timeout' config value" Scenario: Tool agent http_request tool with invalid URL Given a ToolAgent is configured with name "http_invalid_agent" and config diff --git a/robot/version.robot b/robot/version.robot index afa4253..560516c 100644 --- a/robot/version.robot +++ b/robot/version.robot @@ -35,7 +35,7 @@ Core Modules Are Importable [Documentation] Verify core modules import correctly ${result}= Run Process python -c ... from cleveractors.core.config import ConfigurationManager, SchemaValidator\; from cleveractors.core.exceptions import ConfigurationError, AgentCreationError, RoutingError, TemplateError, ExecutionError, ApplicationError, StreamRoutingError\; from cleveractors.core.progress import ProgressBarManager\; print("OK") - ... timeout=5s + ... timeout=30s Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} OK @@ -43,7 +43,7 @@ Templates Modules Are Importable [Documentation] Verify template modules import correctly ${result}= Run Process python -c ... from cleveractors.templates.renderer import TemplateEngine, TemplateRenderer\; from cleveractors.templates.registry import TemplateRegistry\; from cleveractors.templates.enhanced_registry import EnhancedTemplateRegistry\; print("OK") - ... timeout=5s + ... timeout=30s Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} OK @@ -51,6 +51,6 @@ LangGraph Modules Are Importable [Documentation] Verify LangGraph modules import correctly ${result}= Run Process python -c ... from cleveractors.langgraph.graph import LangGraph\; from cleveractors.langgraph.bridge import RxPyLangGraphBridge\; from cleveractors.langgraph.state import GraphState\; from cleveractors.langgraph.nodes import NodeConfig, NodeType\; print("OK") - ... timeout=5s + ... timeout=30s Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} OK diff --git a/src/cleveractors/agents/llm_tools.py b/src/cleveractors/agents/llm_tools.py index 6b3f351..3d37712 100644 --- a/src/cleveractors/agents/llm_tools.py +++ b/src/cleveractors/agents/llm_tools.py @@ -70,7 +70,7 @@ _BUILTIN_TOOL_SCHEMAS: dict[str, dict[str, Any]] = { }, }, "http_request": { - "description": "Make an HTTP request and return the response.", + "description": "Make an HTTP request and return the response. Execution is bounded by the agent's configured `timeout` (seconds, default 1); requests that exceed it are aborted and return a timeout error — increase `timeout` in the tool agent config for slow endpoints.", "parameters": { "type": "object", "properties": { @@ -208,7 +208,7 @@ _BUILTIN_TOOL_SCHEMAS: dict[str, dict[str, Any]] = { }, }, "shell": { - "description": "Execute a shell command and return stdout. Requires allow_shell. Use for: find, grep, cat. Prefer file_read for reading files (has max_chars to prevent context overflow). When creating files in new directories, always use 'mkdir -p && touch '.", + "description": "Execute a shell command and return stdout. Requires allow_shell. Use for: find, grep, cat. Prefer file_read for reading files (has max_chars to prevent context overflow). When creating files in new directories, always use 'mkdir -p && touch '. Execution is bounded by the agent's configured `timeout` (seconds, default 1); commands that exceed it are killed and return a timeout error — increase `timeout` in the tool agent config for long-running commands.", "parameters": { "type": "object", "properties": { diff --git a/src/cleveractors/agents/tool.py b/src/cleveractors/agents/tool.py index 4780963..f36ecd6 100644 --- a/src/cleveractors/agents/tool.py +++ b/src/cleveractors/agents/tool.py @@ -351,7 +351,9 @@ class ToolAgent(Agent): except asyncio.TimeoutError as timeout_err: raise ExecutionError( - f"Command '{command}' timed out after {self.timeout} seconds" + f"Command '{command}' timed out after {self.timeout} seconds. " + "Increase the tool agent's 'timeout' config value (in seconds) " + "if this command legitimately needs more time." ) from timeout_err async def _shell_tool( @@ -603,6 +605,12 @@ class ToolAgent(Agent): ) as response: content = await response.text() return f"Status: {response.status}\n\n{content}" + except asyncio.TimeoutError as timeout_err: + raise ExecutionError( + f"HTTP request to '{url}' timed out after {self.timeout} seconds. " + "Increase the tool agent's 'timeout' config value (in seconds) " + "if this endpoint legitimately needs more time." + ) from timeout_err except Exception as e: raise ExecutionError(f"HTTP request failed: {e}") from e