From e2fc9c96c36b3a7f6f44337ed281b8f02cbeacb6 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Thu, 26 Mar 2026 20:02:55 +0000 Subject: [PATCH] Merge remote-tracking branch 'origin/master' into test/int-wf11-graph-actor --- CHANGELOG.md | 2 + robot/helper_wf11_graph_actor.py | 467 +++++++++++++++++++++++++++++++ robot/wf11_graph_actor.robot | 96 +++++++ 3 files changed, 565 insertions(+) create mode 100644 robot/helper_wf11_graph_actor.py create mode 100644 robot/wf11_graph_actor.robot diff --git a/CHANGELOG.md b/CHANGELOG.md index 707a2dec7..030f4c2e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Added Robot Framework integration test for Specification Workflow Example 11 + (complex graph actor for multi-stage code review, trusted profile). (#775) - Added TDD bug-capture tests for bug #1076 — `use_action()` does not propagate `automation_profile` to Plan. Three Behave BDD scenarios (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence diff --git a/robot/helper_wf11_graph_actor.py b/robot/helper_wf11_graph_actor.py new file mode 100644 index 000000000..39d933b8c --- /dev/null +++ b/robot/helper_wf11_graph_actor.py @@ -0,0 +1,467 @@ +"""Robot Framework helper for Workflow Example 11 — Complex Graph Actor. + +Exercises a custom graph-type actor (5 nodes, 6 edges) for multi-stage +code review with parallel fan-out and result synthesis. Read-only action. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import sys +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import NoReturn + +# --------------------------------------------------------------------------- +# Path bootstrap +# --------------------------------------------------------------------------- +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +_ROBOT = str(Path(__file__).resolve().parent) +if _ROBOT not in sys.path: + sys.path.insert(0, _ROBOT) + +from helper_e2e_common import ( # noqa: E402 + cleanup_workspace, + run_cli, + setup_workspace, + write_yaml, +) + +from cleveragents.actor.compiler import CompiledActor, compile_actor # noqa: E402 +from cleveragents.actor.schema import ActorConfigSchema, ActorType # noqa: E402 + +# --------------------------------------------------------------------------- +# YAML fixtures +# --------------------------------------------------------------------------- + +_GRAPH_ACTOR_YAML = """\ +name: local/wf11-code-reviewer +type: graph +description: >- + Multi-stage code review actor with parallel fan-out to + security, performance, and style reviewer nodes. +version: "1.0" +provider: openai +model: gpt-4 +context_view: reviewer +memory: + enabled: true + max_messages: 50 +route: + nodes: + - id: dispatcher + type: agent + name: Review Dispatcher + description: >- + Receives the code review request and dispatches work to + specialised reviewer sub-actors in parallel. + config: + model: gpt-4 + prompt: >- + You are the review dispatcher. Partition the incoming + code-review request into three independent sub-reviews: + security, performance, and style. + - id: security_reviewer + type: agent + name: Security Reviewer + description: Analyses code for security vulnerabilities and risks. + config: + model: gpt-4 + prompt: >- + You are a security reviewer. Identify OWASP Top-10 + vulnerabilities, injection risks, and credential leaks. + - id: performance_reviewer + type: agent + name: Performance Reviewer + description: Analyses code for performance bottlenecks and regressions. + config: + model: gpt-4 + prompt: >- + You are a performance reviewer. Identify O(n^2) loops, + unnecessary allocations, and missing caching. + - id: style_reviewer + type: agent + name: Style Reviewer + description: Reviews code style, naming, and documentation. + config: + model: gpt-4 + prompt: >- + You are a style reviewer. Check naming conventions, + docstring quality, and PEP-8 compliance. + - id: synthesizer + type: agent + name: Review Synthesizer + description: >- + Collects reports from the three reviewers and produces a + unified review report with prioritised findings. + config: + model: gpt-4 + prompt: >- + You are the review synthesizer. Merge the security, + performance, and style reports into a single unified + review with prioritised findings and an overall verdict. + edges: + - from_node: dispatcher + to_node: security_reviewer + - from_node: dispatcher + to_node: performance_reviewer + - from_node: dispatcher + to_node: style_reviewer + - from_node: security_reviewer + to_node: synthesizer + - from_node: performance_reviewer + to_node: synthesizer + - from_node: style_reviewer + to_node: synthesizer + entry_node: dispatcher + exit_nodes: + - synthesizer +""" + +_ACTION_YAML = """\ +name: local/wf11-code-review +description: >- + Read-only multi-stage code review using a graph actor with + parallel fan-out to security, performance, and style reviewers. +definition_of_done: >- + All three reviewer sub-reports are produced and synthesised into + a unified review report. No files are modified. +strategy_actor: local/wf11-code-reviewer +execution_actor: local/wf11-code-reviewer +read_only: true +""" + + +def _fail(msg: str) -> NoReturn: + print(f"FAIL: {msg}", file=sys.stderr) + raise SystemExit(1) + + +def _extract_plan_id(output: str) -> str | None: + """Extract a ULID plan_id from plain CLI output.""" + match = re.search(r"\b([0-9A-Z]{26})\b", output) + return match.group(1) if match else None + + +# --------------------------------------------------------------------------- +# Common setup helper +# --------------------------------------------------------------------------- + + +@contextlib.contextmanager +def _setup_actor_and_action( + prefix: str, +) -> Iterator[tuple[str, str, str]]: + """Register the graph actor and create the review action. + + Yields ``(workspace, actor_path, action_path)`` and cleans up on exit. + """ + workspace = setup_workspace(prefix=prefix) + actor_path = write_yaml(_GRAPH_ACTOR_YAML) + action_path = write_yaml(_ACTION_YAML) + try: + r1 = run_cli( + "actor", + "add", + "local/wf11-code-reviewer", + "--config", + actor_path, + "--format", + "plain", + workspace=workspace, + ) + if r1.returncode != 0: + _fail(f"actor add: {r1.stderr}") + r2 = run_cli( + "action", + "create", + "--config", + action_path, + workspace=workspace, + ) + if r2.returncode != 0: + _fail( + f"action create rc={r2.returncode}\n" + f"stdout: {r2.stdout}\nstderr: {r2.stderr}" + ) + yield workspace, actor_path, action_path + finally: + os.unlink(actor_path) + os.unlink(action_path) + cleanup_workspace(workspace) + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def register_graph_actor() -> None: + """Register the WF11 graph actor via ``agents actor add``.""" + workspace = setup_workspace(prefix="wf11_reg_") + yaml_path = write_yaml(_GRAPH_ACTOR_YAML) + try: + result = run_cli( + "actor", + "add", + "local/wf11-code-reviewer", + "--config", + yaml_path, + "--format", + "plain", + workspace=workspace, + ) + if result.returncode != 0: + _fail( + f"actor add rc={result.returncode}\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + show = run_cli( + "actor", + "show", + "local/wf11-code-reviewer", + "--format", + "plain", + workspace=workspace, + ) + if show.returncode != 0: + _fail(f"actor show rc={show.returncode}\nstderr: {show.stderr}") + if "local/wf11-code-reviewer" not in show.stdout: + _fail(f"actor not in show output:\n{show.stdout}") + print("wf11-register-graph-actor-ok") + finally: + os.unlink(yaml_path) + cleanup_workspace(workspace) + + +def verify_graph_topology() -> None: + """Parse the graph actor YAML and verify topology (5 nodes, 6 edges).""" + yaml_path = write_yaml(_GRAPH_ACTOR_YAML) + try: + config = ActorConfigSchema.from_yaml_file(yaml_path) + assert config.name == "local/wf11-code-reviewer", f"Wrong name: {config.name}" + assert config.type == ActorType.GRAPH, f"Wrong type: {config.type}" + assert config.route is not None, "route is None" + nodes = config.route.nodes + edges = config.route.edges + assert len(nodes) == 5, f"Expected 5 nodes, got {len(nodes)}" + assert len(edges) == 6, f"Expected 6 edges, got {len(edges)}" + assert config.route.entry_node == "dispatcher" + assert config.route.exit_nodes == ["synthesizer"] + node_ids = {n.id for n in nodes} + expected_ids = { + "dispatcher", + "security_reviewer", + "performance_reviewer", + "style_reviewer", + "synthesizer", + } + assert node_ids == expected_ids, f"Node IDs mismatch: {node_ids}" + fan_out = [e for e in edges if e.from_node == "dispatcher"] + assert len(fan_out) == 3, f"Expected 3 fan-out, got {len(fan_out)}" + fan_in = [e for e in edges if e.to_node == "synthesizer"] + assert len(fan_in) == 3, f"Expected 3 fan-in, got {len(fan_in)}" + print("wf11-verify-graph-topology-ok") + finally: + os.unlink(yaml_path) + + +def compile_graph_actor() -> None: + """Compile the graph actor and verify parallel fan-out metadata.""" + yaml_path = write_yaml(_GRAPH_ACTOR_YAML) + try: + config = ActorConfigSchema.from_yaml_file(yaml_path) + compiled = compile_actor(config) + assert isinstance(compiled, CompiledActor) + assert compiled.name == "local/wf11-code-reviewer" + assert len(compiled.nodes) == 5, f"Expected 5 nodes, got {len(compiled.nodes)}" + assert len(compiled.edges) == 6, f"Expected 6 edges, got {len(compiled.edges)}" + assert compiled.entry_point == "dispatcher" + meta = compiled.metadata + assert len(meta.node_ids) == 5 + assert meta.entry_node == "dispatcher" + assert "synthesizer" in meta.exit_nodes + print("wf11-compile-graph-actor-ok") + finally: + os.unlink(yaml_path) + + +def create_review_action() -> None: + """Create a read-only action referencing the graph actor via CLI.""" + with _setup_actor_and_action("wf11_action_") as (workspace, _actor, _action): + show = run_cli( + "action", + "show", + "local/wf11-code-review", + "--format", + "plain", + workspace=workspace, + ) + if show.returncode != 0: + _fail(f"action show rc={show.returncode}\nstderr: {show.stderr}") + if "local/wf11-code-review" not in show.stdout: + _fail(f"action not in show output:\n{show.stdout}") + # Verify read-only flag + if "yes" not in show.stdout.lower() and "true" not in show.stdout.lower(): + show_json = run_cli( + "action", + "show", + "local/wf11-code-review", + "--format", + "json", + workspace=workspace, + ) + if show_json.returncode == 0: + try: + data = json.loads(show_json.stdout) + if not data.get("read_only"): + _fail(f"action not read_only in JSON: {data}") + except json.JSONDecodeError: + _fail( + "action show --format json returned invalid JSON; " + "read_only verification inconclusive" + ) + print("wf11-create-review-action-ok") + + +def plan_use_review() -> None: + """Create a plan from the review action and verify read-only flag.""" + with _setup_actor_and_action("wf11_plan_") as (workspace, _actor, _action): + r3 = run_cli( + "plan", + "use", + "local/wf11-code-review", + "--automation-profile", + "trusted", + "--format", + "plain", + workspace=workspace, + ) + if r3.returncode != 0: + _fail( + f"plan use rc={r3.returncode}\nstdout: {r3.stdout}\nstderr: {r3.stderr}" + ) + plan_id = _extract_plan_id(r3.stdout) + if not plan_id: + _fail(f"could not extract plan_id from:\n{r3.stdout}") + r4 = run_cli( + "plan", + "status", + plan_id, + "--format", + "plain", + workspace=workspace, + ) + combined = r4.stdout + r4.stderr + if "Traceback" in combined or "INTERNAL" in combined: + _fail(f"plan status crashed:\n{combined}") + print("wf11-plan-use-review-ok") + + +def verify_read_only_guard() -> None: + """Verify ``plan execute`` is rejected for read-only plans.""" + with _setup_actor_and_action("wf11_ro_") as (workspace, _actor, _action): + r3 = run_cli( + "plan", + "use", + "local/wf11-code-review", + "--automation-profile", + "trusted", + "--format", + "plain", + workspace=workspace, + ) + if r3.returncode != 0: + _fail(f"plan use: {r3.stderr}") + plan_id = _extract_plan_id(r3.stdout) + if not plan_id: + _fail(f"could not extract plan_id from:\n{r3.stdout}") + # Attempt plan execute — should be rejected + r4 = run_cli("plan", "execute", plan_id, workspace=workspace) + combined = r4.stdout + r4.stderr + if "Traceback" in combined: + _fail(f"plan execute crashed:\n{combined}") + if r4.returncode == 0: + _fail("Expected non-zero exit for read-only plan") + if "read-only" not in combined.lower() and "read_only" not in combined.lower(): + _fail(f"Expected 'read-only' or 'read_only' in output:\n{combined}") + print("wf11-verify-read-only-guard-ok") + + +def verify_review_synthesis() -> None: + """Verify the graph topology supports unified review synthesis.""" + yaml_path = write_yaml(_GRAPH_ACTOR_YAML) + try: + config = ActorConfigSchema.from_yaml_file(yaml_path) + compiled = compile_actor(config) + incoming = [e for e in compiled.edges if e.target == "synthesizer"] + assert len(incoming) == 3, ( + f"Expected 3 edges into synthesizer, got {len(incoming)}" + ) + sources = {e.source for e in incoming} + expected = {"security_reviewer", "performance_reviewer", "style_reviewer"} + assert sources == expected, f"Reviewer sources mismatch: {sources}" + assert "synthesizer" in compiled.metadata.exit_nodes + assert len(compiled.metadata.exit_nodes) == 1 + no_incoming = [e for e in compiled.edges if e.target == "dispatcher"] + assert len(no_incoming) == 0, "Dispatcher should have 0 incoming edges" + assert config.route is not None + for node in config.route.nodes: + assert node.type.value == "agent", f"Node {node.id} is {node.type.value}" + print("wf11-verify-review-synthesis-ok") + finally: + os.unlink(yaml_path) + + +def verify_no_file_modifications() -> None: + """Verify the action is marked read-only (no file modifications).""" + with _setup_actor_and_action("wf11_nomod_") as (workspace, _actor, _action): + # Use YAML format for structured read_only check + r3 = run_cli( + "action", + "show", + "local/wf11-code-review", + "--format", + "yaml", + workspace=workspace, + ) + if r3.returncode != 0: + _fail( + f"action show rc={r3.returncode}\n" + f"stdout: {r3.stdout}\nstderr: {r3.stderr}" + ) + out = r3.stdout + if "read_only: true" not in out.lower() and "read_only: True" not in out: + _fail(f"Action not marked read_only in YAML output:\n{out}") + if "local/wf11-code-reviewer" not in out: + _fail(f"Graph actor reference not found in output:\n{out}") + print("wf11-verify-no-file-modifications-ok") + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], None]] = { + "register-graph-actor": register_graph_actor, + "verify-graph-topology": verify_graph_topology, + "compile-graph-actor": compile_graph_actor, + "create-review-action": create_review_action, + "plan-use-review": plan_use_review, + "verify-read-only-guard": verify_read_only_guard, + "verify-review-synthesis": verify_review_synthesis, + "verify-no-file-modifications": verify_no_file_modifications, +} + +if __name__ == "__main__": + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>") + sys.exit(1) + fn = _COMMANDS[sys.argv[1]] + fn() diff --git a/robot/wf11_graph_actor.robot b/robot/wf11_graph_actor.robot new file mode 100644 index 000000000..da54db806 --- /dev/null +++ b/robot/wf11_graph_actor.robot @@ -0,0 +1,96 @@ +*** Settings *** +Documentation Workflow Example 11 — Complex Graph Actor for Multi-Stage +... Code Review. Exercises a custom graph-type actor (5 nodes, +... 6 edges) with parallel fan-out to security, performance, +... and style reviewer nodes and result synthesis. The action +... is read-only. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment +Force Tags wf11 graph trusted integration + +*** Variables *** +${HELPER} ${CURDIR}/helper_wf11_graph_actor.py + +*** Test Cases *** +Register Graph Actor Via CLI + [Documentation] Register the 5-node graph actor via + ... ``agents actor add --config`` and verify persistence + ... with ``actor show``. + ${result}= Run Process ${PYTHON} ${HELPER} register-graph-actor cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-register-graph-actor-ok + +Verify Graph Topology 5 Nodes 6 Edges + [Documentation] Parse the graph actor YAML and verify topology: + ... 5 nodes, 6 edges, entry = dispatcher, + ... exit = synthesizer, 3-way fan-out/fan-in. + ${result}= Run Process ${PYTHON} ${HELPER} verify-graph-topology cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-verify-graph-topology-ok + +Compile Graph Actor To LangGraph StateGraph + [Documentation] Compile the graph actor via the actor compiler. + ... Verifies 5 compiled nodes, 6 compiled edges, + ... correct entry point, and compilation metadata. + ${result}= Run Process ${PYTHON} ${HELPER} compile-graph-actor cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-compile-graph-actor-ok + +Create Read Only Review Action + [Documentation] Create a read-only action referencing the graph + ... actor as both strategy and execution actor. + ... Verifies action persistence and read-only flag. + ${result}= Run Process ${PYTHON} ${HELPER} create-review-action cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-create-review-action-ok + +Plan Use Review With Graph Actor + [Documentation] Create a plan from the read-only review action. + ... Verifies plan creation succeeds and plan status + ... reports strategize/queued without errors. + ${result}= Run Process ${PYTHON} ${HELPER} plan-use-review cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-plan-use-review-ok + +Verify Read Only Guard On Plan Execute + [Documentation] Verify that ``plan execute`` does not crash for + ... a read-only plan. The command should either reject + ... with a read-only message or report the plan is not + ... ready — never a traceback. + ${result}= Run Process ${PYTHON} ${HELPER} verify-read-only-guard cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-verify-read-only-guard-ok + +Verify Review Synthesis Structure + [Documentation] Verify the graph topology supports unified review + ... synthesis: synthesizer receives from exactly 3 + ... reviewer nodes and is the sole exit, confirming + ... the merged review output structure. + ${result}= Run Process ${PYTHON} ${HELPER} verify-review-synthesis cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-verify-review-synthesis-ok + +Verify No File Modifications Via Read Only Action + [Documentation] Verify the action is marked read-only in YAML + ... output, confirming no file modifications. Also + ... verifies the correct graph actor references. + ${result}= Run Process ${PYTHON} ${HELPER} verify-no-file-modifications cwd=${SUITE_HOME} timeout=120s on_timeout=kill + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} wf11-verify-no-file-modifications-ok -- 2.52.0