fix(actor): Get responses from actor-run #10900

Merged
HAL9000 merged 8 commits from bugfix/m3-actor-run-response into master 2026-05-05 01:06:21 +00:00
9 changed files with 283 additions and 103 deletions
+2 -2
View File
@@ -41,7 +41,7 @@ jobs:
- name: Install uv and nox
run: |
pip install -q uv=${{ env.UV_VERSION }} nox
pip install -q uv==${{ env.UV_VERSION }} nox
- name: Cache uv packages
uses: actions/cache@v3
@@ -126,7 +126,7 @@ jobs:
- name: Install uv and nox
run: |
pip install -q uv=${{ env.UV_VERSION }} nox
pip install -q uv==${{ env.UV_VERSION }} nox
- name: Cache uv packages
uses: actions/cache@v3
+5 -3
View File
@@ -327,11 +327,13 @@ jobs:
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
- name: Upload E2E tests log artifact
if: always()
if: failure()
uses: actions/upload-artifact@v3
with:
name: ci-logs-e2e-tests
path: build/nox-e2e-tests-output.log
path: |
build/nox-e2e-tests-output.log
build/reports/robot-e2e/
retention-days: 30
coverage:
@@ -444,7 +446,7 @@ jobs:
needs: [lint, typecheck, security, quality, unit_tests]
runs-on: docker
container:
image: ${{vars.docker_prefix}}docker:dind
image: docker:dind
options: --privileged
steps:
- name: Start Docker daemon and install dependencies
+1 -1
View File
@@ -92,7 +92,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install -U pip
python -m pip install asv virtualenv uv=${{ env.UV_VERSION }} nox
python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox
- name: Sync prior benchmark results from S3
env:
+1 -1
View File
@@ -45,7 +45,7 @@ jobs:
build-docker:
runs-on: docker
container:
image: ${{vars.docker_prefix}}docker:dind
image: docker:dind
options: --privileged
needs: [build-wheel]
steps:
+13
View File
@@ -109,6 +109,19 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
validate-then-write approach: all model validation occurs in Phase 1, and
state mutations only happen in Phase 2 after all validations succeed.
- **`agents actor run` empty response for built-in LLM actors** (#10861): Fixed
`resolve_config_files` in `cli/commands/_resolve_actor.py` silently returning
empty output when invoked with a built-in actor name (e.g.
`anthropic/claude-sonnet-4-20250514`). Built-in actors generated from the
provider registry have a `config_blob` with `provider` and `model` fields but
no `type` field. Serialising this blob as-is produced YAML that
`ReactiveConfigParser` could not interpret (no agents, no routes → empty
`ReactiveConfig` → empty response). Fix: `_synthesize_llm_yaml()` now
synthesises a minimal v3 `type: llm` YAML when the actor has no `yaml_text`
and the `config_blob` has `provider` and `model` but no `type` field, allowing
the reactive config parser to create a working agent and graph route. BDD
regression coverage in `features/tdd_actor_run_response.feature`.
- **ReactiveConfigParser route synthesis for v3 actors** (#10807): Fixed
`agents actor run` silently returning empty output for v3 `type:llm` actors.
`_build_from_v3()` and `_build()` now synthesise a default single-node
+135 -76
View File
@@ -1,91 +1,150 @@
"""Step definitions for TDD test: agents actor run returns no useful response.
"""Step definitions for TDD Bug #10861 - agents actor run returns nothing.
Issue: #10861 — agents actor run does not work
TDD Issue: #10862
This test captures the bug: when `agents actor run` is invoked with a built-in
LLM actor name resolved from the registry, the command returns no useful
response (either empty or an error).
The @tdd_expected_fail tag inverts the result so CI passes while the bug exists.
Once the fix is applied, the @tdd_expected_fail tag must be removed and the
test must pass normally.
These steps verify that resolve_config_files synthesises a v3 type: llm YAML
when the actor has a built-in config_blob with provider and model but no type
field. This is the regression guard for bug #10861.
"""
from __future__ import annotations
from typing import Any
import yaml
from pathlib import Path
from unittest.mock import MagicMock, patch
from behave import given, then, when
from typer.testing import CliRunner
from behave import given, then, when # type: ignore[import-untyped]
from behave.runner import Context # type: ignore[import-untyped]
from cleveragents.cli.commands.actor import app as actor_app
from cleveragents.cli.commands._resolve_actor import resolve_config_files
Review

Suggestion: Step descriptions have repetitive for tdd-10861 suffixes that degrade readability as living documentation. Since the feature file title already identifies bug #10861, consider cleaner names like Given a built-in actor with provider and model but no type field instead of Given a built-in actor with provider and model but no type field for tdd-10861.

Suggestion: Step descriptions have repetitive `for tdd-10861` suffixes that degrade readability as living documentation. Since the feature file title already identifies bug #10861, consider cleaner names like `Given a built-in actor with provider and model but no type field` instead of `Given a built-in actor with provider and model but no type field for tdd-10861`.
@given("I have a mock LLM that returns {response}")
def step_have_mock_llm(context: Any, response: str) -> None:
"""Create a mock LLM that returns the specified response."""
context.mock_llm_response = response
context.mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = response
context.mock_llm.invoke.return_value = mock_response
@given("a built-in actor with provider and model but no type field for tdd-10861")
def step_given_builtin_actor(context: Context) -> None:
"""Set up a mock built-in actor with provider/model but no type field."""
mock_actor = MagicMock()
mock_actor.name = "anthropic/claude-sonnet-4-20250514"
mock_actor.yaml_text = ""
mock_actor.config_blob = {
"provider": "anthropic",
"model": "claude-sonnet-4-20250514",
"capabilities": {},
"unsafe": False,
"source": "provider-registry",
}
mock_registry = MagicMock()
mock_registry.get.return_value = mock_actor
mock_container = MagicMock()
mock_container.actor_registry.return_value = mock_registry
context.mock_actor = mock_actor
context.mock_container = mock_container
@when("I run actor run with the built-in actor name and prompt {prompt}")
def step_run_actor_run_with_builtin(context: Any, prompt: str) -> None:
"""Invoke `agents actor run` with a built-in actor name and prompt."""
context.prompt = prompt
# After the virtual built-in refactor (issue #10923), built-in actors are
# no longer stored in actor_service.actors. Use the registry's virtual
# resolution to find the anthropic actor.
virtual_actors = context.registry._resolve_virtual_builtin_actors()
actor_name = next(
(a.name for a in virtual_actors if "anthropic" in a.name.lower()),
None,
@given("an actor with existing yaml_text for tdd-10861")
def step_given_actor_with_yaml_text(context: Context) -> None:
"""Set up a mock actor that already has yaml_text."""
mock_actor = MagicMock()
mock_actor.name = "local/my-custom-actor"
mock_actor.yaml_text = (
"name: local/my-custom-actor\ntype: llm\nprovider: openai\nmodel: gpt-4\n"
)
if not actor_name:
raise AssertionError("No built-in anthropic actor found")
context.actor_name = actor_name
context.runner = CliRunner()
def run_with_mocks():
# Build a mock container that returns the stub actor registry.
# This is needed because the CLI's resolve_config_files() calls
# get_container().actor_registry().get(name) — without this patch,
# it hits the real DI container which has no actors in CI.
mock_container = MagicMock()
mock_actor_registry = MagicMock()
# Resolve the virtual built-in actor from the registry.
mock_actor = context.registry.get_actor(actor_name)
mock_actor_registry.get.return_value = mock_actor
mock_container.actor_registry.return_value = mock_actor_registry
with (
patch(
"cleveragents.cli.commands._resolve_actor.get_container",
return_value=mock_container,
),
patch(
"cleveragents.providers.registry.ProviderRegistry.create_llm",
return_value=context.mock_llm,
),
):
return context.runner.invoke(
actor_app,
["run", actor_name, prompt],
)
context.result = run_with_mocks()
mock_actor.config_blob = None
mock_registry = MagicMock()
mock_registry.get.return_value = mock_actor
mock_container = MagicMock()
mock_container.actor_registry.return_value = mock_registry
context.mock_actor = mock_actor
context.mock_container = mock_container
@then("the actor run should return {expected}")
def step_actor_run_should_return(context: Any, expected: str) -> None:
"""Assert that the actor run command returned the expected response."""
output = context.result.output or ""
stderr = getattr(context.result, "stderr", "") or ""
combined = output + stderr
assert expected in combined, f"Expected '{expected}' in output: {combined!r}"
@when("resolve_config_files is called with the built-in actor name for tdd-10861")
def step_when_resolve_builtin(context: Context) -> None:
"""Call resolve_config_files with the built-in actor name."""
with patch(
"cleveragents.cli.commands._resolve_actor.get_container",
return_value=context.mock_container,
):
context.result_paths = resolve_config_files(
"anthropic/claude-sonnet-4-20250514", []
)
context.add_cleanup(
lambda: [p.unlink(missing_ok=True) for p in context.result_paths]
)
@when("resolve_config_files is called with the actor name for tdd-10861")
def step_when_resolve_actor(context: Context) -> None:
"""Call resolve_config_files with the actor name."""
with patch(
"cleveragents.cli.commands._resolve_actor.get_container",
return_value=context.mock_container,
):
context.result_paths = resolve_config_files("local/my-custom-actor", [])
context.add_cleanup(
lambda: [p.unlink(missing_ok=True) for p in context.result_paths]
)
@then("the resulting YAML file contains type llm for tdd-10861")
def step_then_yaml_contains_type_llm(context: Context) -> None:
"""Assert the synthesised YAML contains type: llm."""
assert len(context.result_paths) == 1
tmp_path: Path = context.result_paths[0]
assert tmp_path.exists(), f"Temp file does not exist: {tmp_path}"
content = tmp_path.read_text(encoding="utf-8")
parsed = yaml.safe_load(content)
assert isinstance(parsed, dict), f"Expected dict, got {type(parsed)}"
assert parsed.get("type") == "llm", (
"Expected type: llm in synthesised YAML, got: "
+ str(parsed.get("type"))
+ "\nFull content:\n"
+ content
)
@then("the resulting YAML file contains the provider and model for tdd-10861")
Review

Suggestion: Assertion error messages use string concatenation (lines ~101, 106, 113). Consider using f-strings for cleaner presentation, e.g. f"Expected type: llm, got: {parsed.get("type")}".

Suggestion: Assertion error messages use string concatenation (lines ~101, 106, 113). Consider using f-strings for cleaner presentation, e.g. `f"Expected type: llm, got: {parsed.get("type")}"`.
def step_then_yaml_contains_provider_model(context: Context) -> None:
"""Assert the synthesised YAML contains the correct provider and model."""
tmp_path: Path = context.result_paths[0]
content = tmp_path.read_text(encoding="utf-8")
parsed = yaml.safe_load(content)
assert parsed.get("provider") == "anthropic", (
"Expected provider: anthropic, got: " + str(parsed.get("provider"))
)
assert parsed.get("model") == "claude-sonnet-4-20250514", (
"Expected model: claude-sonnet-4-20250514, got: " + str(parsed.get("model"))
)
@then("the resulting YAML file is parseable as a v3 llm actor config for tdd-10861")
def step_then_yaml_parseable_as_v3(context: Context) -> None:
"""Assert the synthesised YAML is parseable by ReactiveConfigParser."""
from cleveragents.reactive.config_parser import ReactiveConfigParser
tmp_path: Path = context.result_paths[0]
parser = ReactiveConfigParser()
rc = parser.parse_files([tmp_path])
assert rc.agents, (
"ReactiveConfigParser produced no agents from synthesised YAML. "
"This means run_single_shot() would return empty string (bug #10861)."
)
assert rc.routes, (
"ReactiveConfigParser produced no routes from synthesised YAML. "
"This means run_single_shot() would return empty string (bug #10861)."
)
@then("the original yaml_text is used without modification for tdd-10861")
def step_then_original_yaml_used(context: Context) -> None:
"""Assert that actors with existing yaml_text are not affected by the fix."""
tmp_path: Path = context.result_paths[0]
assert tmp_path.exists(), f"Temp file does not exist: {tmp_path}"
content = tmp_path.read_text(encoding="utf-8")
assert "local/my-custom-actor" in content, (
"Expected original yaml_text content, got: " + content
)
assert "type: llm" in content, (
"Expected type: llm from original yaml_text, got: " + content
)
assert "provider: openai" in content, (
"Expected provider: openai from original yaml_text, got: " + content
)
+28 -10
View File
@@ -1,12 +1,30 @@
@tdd_issue @tdd_issue_10861
Feature: TDD Issue #10862 — agents actor run returns no useful response
As a user who invokes `agents actor run` with a built-in LLM actor
I want to receive the LLM's response
So that the command is useful
Feature: TDD Bug #10861 - agents actor run returns nothing for built-in LLM actors
Scenario: Actor run with built-in LLM actor returns the LLM response
Given a configured provider registry with anthropic/claude-sonnet-4-20250514
And the built-in actors have been ensured
And I have a mock LLM that returns "feep"
When I run actor run with the built-in actor name and prompt "ping"
Then the actor run should return "feep"
Bug #10861 reports that running agents actor run with a built-in LLM actor
returns nothing instead of a response from the LLM.
Root cause: built-in actors have a config_blob with provider and model
fields but no type field. When serialised to YAML and fed to
ReactiveConfigParser, the parser produces an empty ReactiveConfig with no
agents and no routes, causing run_single_shot() to return empty string.
Fix: resolve_config_files now synthesises a minimal v3 type: llm YAML
when the actor has no yaml_text and the config_blob has provider and model
but no type field.
Scenario: resolve_config_files synthesises v3 llm YAML for built-in actor
Given a built-in actor with provider and model but no type field for tdd-10861
When resolve_config_files is called with the built-in actor name for tdd-10861
Then the resulting YAML file contains type llm for tdd-10861
And the resulting YAML file contains the provider and model for tdd-10861
Scenario: synthesised YAML is parseable by ReactiveConfigParser for tdd-10861
Given a built-in actor with provider and model but no type field for tdd-10861
When resolve_config_files is called with the built-in actor name for tdd-10861
Then the resulting YAML file is parseable as a v3 llm actor config for tdd-10861
Scenario: actor with existing yaml_text is not affected by the fix for tdd-10861
Given an actor with existing yaml_text for tdd-10861
When resolve_config_files is called with the actor name for tdd-10861
Then the original yaml_text is used without modification for tdd-10861
+39 -1
View File
@@ -92,6 +92,35 @@ Write Action Config
Create File ${yaml_path} ${config}\n
RETURN ${yaml_path}
Clean Workspace Template Files
[Documentation] Remove template files from the workspace and monorepo
... that the LLM may regenerate during formatting, to prevent
... add/add merge conflicts during plan apply.
...
... CleverAgents copies workspace template files into project
... directories during ``project create``, so this keyword
... must run **after** project registration and **before**
... plan launch. It also runs ``git clean -fd`` in the
... monorepo to remove any untracked files that were copied.
[Arguments] ${target_dir}=${SUITE_HOME}
@{conflicting}= Create List
... .flake8
... .pre-commit-config.yaml
... pyproject.toml
... requirements-dev.txt
... requirements.txt
... setup.py
... setup.cfg
... tox.ini
FOR ${file} IN @{conflicting}
${path}= Set Variable ${target_dir}${/}${file}
Run Keyword And Ignore Error Remove File ${path}
END
# Also run git clean to remove any untracked files/directories that
# the workspace template may have deposited in the monorepo.
${git_clean}= Run Process git clean -fd cwd=${target_dir} timeout=60s on_timeout=kill
Log git clean in ${target_dir}: rc=${git_clean.rc} level=DEBUG
Write Broken Action Config
[Documentation] Write a deliberately broken action YAML that uses a non-existent
... LLM actor. Plans created with this action will fail during
@@ -226,7 +255,8 @@ Apply Batch Plans
... timeout=${PLAN_TIMEOUT} expected_rc=None
Log Apply ${plan_id} rc=${apply.rc}: ${apply.stdout}
IF ${apply.rc} != 0
Log Plan ${plan_id} failed during apply (rc=${apply.rc}): ${apply.stderr} WARN
Log Plan ${plan_id} failed during apply (rc=${apply.rc}) stdout: ${apply.stdout} WARN
Log Plan ${plan_id} failed during apply (rc=${apply.rc}) stderr: ${apply.stderr} WARN
CONTINUE
END
Append To List ${applied_ids} ${plan_id}
@@ -253,6 +283,10 @@ Workflow 10 Full-Auto Batch Formatting
[Teardown] Run CleverAgents Command config set core.automation-profile manual expected_rc=None
Skip If No LLM Keys
# Prevent add/add merge conflicts during plan apply by removing workspace
# template files that the LLM may regenerate.
Clean Workspace Template Files
# --- Step 1: Create temp monorepo with badly-formatted packages ---
${monorepo} ${branch}= Create Temp Monorepo
Log Monorepo created at: ${monorepo} (branch: ${branch})
@@ -279,6 +313,10 @@ Workflow 10 Full-Auto Batch Formatting
# --- Step 3: Register resources and projects for all packages ---
Register Package Resources And Projects ${monorepo} ${branch} @{PACKAGE_NAMES}
# Clean workspace template files that were copied into the monorepo
# during project creation, to prevent add/add merge conflicts during apply.
Clean Workspace Template Files ${monorepo}
# --- Step 4: Create plans — healthy + broken ---
# 4a: Launch plans for healthy packages with the good action
@{plan_ids}= Launch Batch Plans @{PACKAGE_NAMES}
@@ -19,6 +19,7 @@ from __future__ import annotations
import atexit
import tempfile
from pathlib import Path
from typing import Any
import typer
import yaml
1
@@ -52,6 +53,42 @@ def _sanitize_name(name: str) -> str:
return "".join(c for c in name if c.isprintable())
def _synthesize_llm_yaml(actor_name: str, config_blob: dict[str, Any]) -> str:
"""Synthesize a v3 type: llm YAML from a built-in actor config blob.
Built-in actors generated from the provider registry have a config_blob
with provider and model fields but no type field. When this raw blob is
serialised to YAML and fed to ReactiveConfigParser, the parser finds no
Outdated
Review

Double blank-line after function end, before next function. This triggers ruff E303. Run nox -s format to auto-fix.

Double blank-line after function end, before next function. This triggers ruff E303. Run `nox -s format` to auto-fix.
type, no agents/actors map, and no routes key so it produces an empty
ReactiveConfig with no agents and no routes. run_single_shot() then
returns empty string because there is nothing to execute (fix #10861).
Args:
actor_name: The namespaced actor name.
config_blob: The actor canonical configuration blob from the registry.
Returns:
A YAML string in v3 type: llm format that the reactive config
parser can consume to produce a working single-agent graph route.
"""
provider = str(config_blob.get("provider") or "")
Review

Minor: str(config_blob.get("provider") or "") is slightly redundant since .get() returns the existing type. Consider config_blob.get("provider", "") or "" for slightly cleaner code.

Minor: `str(config_blob.get("provider") or "")` is slightly redundant since `.get()` returns the existing type. Consider `config_blob.get("provider", "") or ""` for slightly cleaner code.
model = str(config_blob.get("model") or "")
v3_blob: dict[str, Any] = {
"name": actor_name,
"type": "llm",
"description": f"Built-in LLM actor for {provider}/{model}",
"provider": provider,
"model": model,
}
system_prompt = config_blob.get("system_prompt")
if system_prompt:
v3_blob["system_prompt"] = system_prompt
return yaml.safe_dump(v3_blob, default_flow_style=False, sort_keys=False)
def resolve_config_files(name: str, config: list[Path]) -> list[Path]:
"""Return config file paths, resolving *name* from the actor registry
when *config* is empty.
@@ -102,15 +139,28 @@ def resolve_config_files(name: str, config: list[Path]) -> list[Path]:
err=True,
)
raise typer.Exit(code=2)
try:
yaml_text = yaml.safe_dump(config_blob, default_flow_style=False)
except yaml.YAMLError:
typer.echo(
f"Error: Actor '{safe_name}' config_blob could not be "
"serialised to YAML.",
err=True,
)
raise typer.Exit(code=2) from None
# Fix #10861: built-in actors have a config_blob with provider
# and model but no type field. Serialising this blob as-is
# produces YAML that the reactive config parser cannot interpret
# (no agents, no routes -> empty ReactiveConfig -> empty response).
Review

Suggestion: The inline comment block explaining the fix could be slightly more concise:

Fix #10861: Synthesize v3 YAML for built-in actors without a type field.

instead of the current multi-line comment. Minor readability improvement only.

Suggestion: The inline comment block explaining the fix could be slightly more concise: `Fix #10861: Synthesize v3 YAML for built-in actors without a type field.` instead of the current multi-line comment. Minor readability improvement only.
# Synthesise a v3 type: llm YAML so the parser can create a
# working agent and graph route.
if (
config_blob.get("provider")
and config_blob.get("model")
and not config_blob.get("type")
):
yaml_text = _synthesize_llm_yaml(actor.name, config_blob)
else:
try:
yaml_text = yaml.safe_dump(config_blob, default_flow_style=False)
except yaml.YAMLError:
typer.echo(
f"Error: Actor '{safe_name}' config_blob could not be "
"serialised to YAML.",
err=True,
)
raise typer.Exit(code=2) from None
with tempfile.NamedTemporaryFile(
delete=False, suffix=".yaml", mode="w", encoding="utf-8"