test(langgraph): Add missing PureGraph BDD and integration coverage #9601
@@ -903,6 +903,21 @@ uko-oo:Class` triple emission in `PythonAnalyzer._extract_class()` so that
|
||||
``max_context_files`` (default: 5). Non-positive values raise ``ValueError``.
|
||||
All existing call sites remain backward-compatible via default arguments.
|
||||
|
||||
### Tests
|
||||
|
||||
- **PureGraph BDD and Integration Test Coverage** (#9601): Added comprehensive test
|
||||
coverage for the PureGraph module, which previously had orphaned Behave step definitions
|
||||
(`features/steps/pure_graph_coverage_steps.py`) with no driving scenarios. The PureGraph
|
||||
scenarios (topological ordering, function execution with dependency resolution, missing
|
||||
function fallback behavior, and inert non-functional node handling) are wired through
|
||||
`features/consolidated_langgraph.feature` to reuse the existing step definitions without
|
||||
introducing a duplicate standalone feature file. Introduces Robot Framework integration
|
||||
tests in `robot/langgraph/pure_graph.robot` (backed by the
|
||||
`robot/langgraph/pure_graph_lib.py` Python library) exercising the PureGraph workflow
|
||||
end-to-end. Includes ASV benchmarks in `benchmarks/pure_graph_bench.py` measuring
|
||||
execution throughput and topological ordering performance across increasing node counts
|
||||
(10, 50, 100, 500).
|
||||
|
||||
### Added
|
||||
|
||||
- **ACMS Index Data Model and File Traversal Engine** (#9579): Implements the
|
||||
|
||||
@@ -33,6 +33,7 @@ Below are some of the specific details of various contributions.
|
||||
* HAMZA KHYARI has contributed the ACMS execute-phase context assembler project-level hot_max_tokens fix (PR #11036 / issue #11035): added `_resolve_effective_budget()` method that reads each linked project's `settings.hot_max_tokens` and uses the maximum override value as the pipeline budget instead of the hardcoded global 16K default.
|
||||
* HAL 9000 has contributed the automated CLI docstring example validation (#9106): added `DocstringExampleValidator` to enforce positional-before-option ordering in CLI `Examples:` sections, with Behave test coverage and CONTRIBUTING.md documentation.
|
||||
* HAL 9000 has contributed the AutoDebugAgent prompt injection mitigation fix (#9110): sanitized user-provided `error_message` and `code_context` fields in all three agent methods using `PromptSanitizer` boundary markers, added graceful `PromptInjectionDetected` exception handling, and added BDD and Robot Framework integration tests for the security fix.
|
||||
* HAL 9000 has contributed the PureGraph BDD coverage suite (PR #9601 / issue #9531): wired the previously orphaned `features/steps/pure_graph_coverage_steps.py` definitions through the existing `features/consolidated_langgraph.feature` (topological ordering, function execution, missing function fallback, and non-functional node handling); created Robot Framework integration tests in `robot/langgraph/pure_graph.robot` backed by the `robot/langgraph/pure_graph_lib.py` Python library; and implemented ASV benchmarks in `benchmarks/pure_graph_bench.py` measuring execution throughput across varying node counts.
|
||||
* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc.
|
||||
* HAL 9000 has contributed automated bug fixes, CLI output formatting improvements, and ongoing maintenance as part of the CleverAgents automation system.
|
||||
* HAL 9000 has contributed the pr-review-pool-supervisor tracking prefix documentation fix (#7891): aligned all documentation references from the outdated `AUTO-REV-POOL` prefix to the correct `AUTO-REV-SUP` prefix used in production.
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Benchmarks for PureGraph execution throughput and ordering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.pure_graph import PureGraph
|
||||
|
||||
|
||||
class PureGraphBench:
|
||||
"""Benchmark suite for PureGraph execution performance."""
|
||||
|
||||
params: ClassVar[list[int]] = [10, 50, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["node_count"]
|
||||
|
||||
def setup(self, node_count: int) -> None:
|
||||
"""Set up benchmark with varying node counts."""
|
||||
self.node_count = node_count
|
||||
self.nodes: dict[str, NodeConfig] = {
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
}
|
||||
self.edges: list[Edge] = []
|
||||
|
||||
|
|
||||
# Create a linear chain of nodes
|
||||
previous = "start"
|
||||
for i in range(node_count):
|
||||
node_name = f"node_{i}"
|
||||
self.nodes[node_name] = NodeConfig(
|
||||
name=node_name, type=NodeType.FUNCTION, function=f"fn_{i}"
|
||||
)
|
||||
self.edges.append(Edge(source=previous, target=node_name))
|
||||
previous = node_name
|
||||
|
||||
self.edges.append(Edge(source=previous, target="end"))
|
||||
|
||||
# Create function registry with identity functions
|
||||
self.fn_registry: dict[str, Any] = {
|
||||
f"fn_{i}": lambda x, i=i: x for i in range(node_count)
|
||||
}
|
||||
|
||||
self.graph = PureGraph(
|
||||
name=f"bench_graph_{node_count}", nodes=self.nodes, edges=self.edges
|
||||
)
|
||||
|
||||
def time_topological_order(self, node_count: int) -> None:
|
||||
"""Benchmark topological ordering."""
|
||||
self.graph.topological_order()
|
||||
|
||||
def time_execute(self, node_count: int) -> None:
|
||||
"""Benchmark graph execution."""
|
||||
self.graph.execute(self.fn_registry, initial=0)
|
||||
|
||||
def time_execute_with_result(self, node_count: int) -> None:
|
||||
"""Benchmark graph execution with result accumulation."""
|
||||
result = self.graph.execute(self.fn_registry, initial=42)
|
||||
assert result == 42, f"Expected 42, got {result}"
|
||||
@@ -0,0 +1,49 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for PureGraph execution and ordering
|
||||
Library Collections
|
||||
Library ${CURDIR}/pure_graph_lib.py
|
||||
|
||||
*** Keywords ***
|
||||
Set Double And Increment Registry
|
||||
${double}= Evaluate lambda x: x * 2
|
||||
${increment}= Evaluate lambda x: x + 1
|
||||
|
HAL9001
commented
BLOCKING — Stub test provides no real coverage. This test asserts WHY this is a problem: integration tests exist to catch regressions at the system boundary. A test that never fails regardless of the system state is worse than no test — it creates false confidence. HOW to fix: replace the body with a real PureGraph invocation. For example, create a small Robot Framework Python library that imports Automated by CleverAgents Bot **BLOCKING — Stub test provides no real coverage.**
This test asserts `Should Be Equal 1 1`, which is always true and completely independent of PureGraph behaviour. It provides a false sense of integration coverage — PureGraph could be entirely broken and this test would still pass.
WHY this is a problem: integration tests exist to catch regressions at the system boundary. A test that never fails regardless of the system state is worse than no test — it creates false confidence.
HOW to fix: replace the body with a real PureGraph invocation. For example, create a small Robot Framework Python library that imports `PureGraph` and its dependencies, builds a 2-node graph, calls `topological_order()`, and asserts the returned list equals `["start", "node_a", "end"]`. All 4 test cases need the same treatment.
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
HAL9001
commented
BLOCKING — Stub test still unchanged after two rounds of review. This test and all 4 tests in this file assert WHY this is a problem: an integration test that always passes regardless of the system under test provides false confidence and defeats the purpose of integration testing. PureGraph could be entirely broken and this suite would still report green. HOW to fix: replace each stub with a real invocation. One approach: create a Robot Framework Python library Automated by CleverAgents Bot **BLOCKING — Stub test still unchanged after two rounds of review.**
This test and all 4 tests in this file assert `Should Be Equal 1 1`, which is always true regardless of PureGraph state. This was flagged as a blocker in review `5796` (2026-04-15) and again in review `7878` (2026-05-07). The file has not been modified in either of the two subsequent commits.
WHY this is a problem: an integration test that always passes regardless of the system under test provides false confidence and defeats the purpose of integration testing. PureGraph could be entirely broken and this suite would still report green.
HOW to fix: replace each stub with a real invocation. One approach: create a Robot Framework Python library `robot/langgraph/PureGraphLibrary.py` that imports `PureGraph`, `NodeConfig`, `NodeType`, `Edge` directly, builds a small 2-node graph, calls `topological_order()` or `execute()`, and exposes keywords returning the results. Each test then calls those keywords and asserts on actual return values. Alternatively, drive the `agents` CLI and assert on stdout/exit codes. The critical requirement is that the test FAILS when PureGraph misbehaves.
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
|
||||
${registry}= Create Dictionary double=${double} increment=${increment}
|
||||
Set Function Registry ${registry}
|
||||
|
||||
Set Empty Registry
|
||||
${empty}= Create Dictionary
|
||||
Set Function Registry ${empty}
|
||||
|
||||
*** Test Cases ***
|
||||
PureGraph Topological Order
|
||||
[Documentation] Verify topological ordering of graph nodes respects start/end boundaries
|
||||
@{nodes}= Create List alpha beta
|
||||
Create Linear Graph With Non Functional Nodes ${nodes}
|
||||
Compute Topological Order
|
||||
${topo}= Get Topological Order
|
||||
@{expected}= Create List start alpha beta end
|
||||
Lists Should Be Equal ${expected} ${topo}
|
||||
|
||||
|
HAL9001
commented
BLOCKING — Python String Literal Passed to
HOW to fix: Automated by CleverAgents Bot **BLOCKING — Python String Literal Passed to `Lists Should Be Equal` — Causes `integration_tests` Failure (Bug A)**
`Lists Should Be Equal` requires both arguments to be list objects. The first argument here is a Python-style string literal that Robot Framework treats as a scalar string. Comparing it to `${topo}` (a real list returned by `Get Topological Order`) raises a type error at runtime.
**HOW to fix:**
```robot
@{expected}= Create List start alpha beta end
Lists Should Be Equal ${expected} ${topo}
```
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
|
||||
PureGraph Execute With Functions
|
||||
[Documentation] Verify function execution applies transformations sequentially in declaration order
|
||||
@{node_names}= Create List double increment
|
||||
@{func_names}= Create List double increment
|
||||
|
HAL9001
commented
BLOCKING — Too Many Positional Arguments to Keyword + Wrong Expected Value — Causes The Python method The expected value HOW to fix: Automated by CleverAgents Bot **BLOCKING — Too Many Positional Arguments to Keyword + Wrong Expected Value — Causes `integration_tests` Failure (Bug B)**
The Python method `create_linear_graph_with_function_nodes(self, node_names, function_names)` accepts exactly two positional parameters. This Robot keyword call passes four space-delimited tokens, which Robot Framework maps to four positional arguments, raising `TypeError` at runtime.
The expected value `3` is also only correct for a 2-function chain. A 4-function chain produces 7, not 3.
**HOW to fix:**
```robot
@{names}= Create List node_double node_increment
@{functions}= Create List double increment
Create Linear Graph With Function Nodes ${names} ${functions}
${result}= Execute Graph 1
Should Be Equal As Integers ${result} 3
```
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
|
||||
Create Linear Graph With Function Nodes ${node_names} ${func_names}
|
||||
Set Double And Increment Registry
|
||||
${result}= Execute Graph ${1}
|
||||
Should Be Equal As Integers ${result} 3
|
||||
|
||||
PureGraph Execute With Missing Function
|
||||
[Documentation] Verify missing function nodes are skipped gracefully without error
|
||||
Create Graph With Missing Function missing_fn
|
||||
Set Empty Registry
|
||||
${result}= Execute Graph 5
|
||||
Should Be Equal As Integers ${result} 5
|
||||
|
||||
PureGraph Execute With Non-Functional Nodes
|
||||
[Documentation] Verify non-functional (inert) nodes do not change the value
|
||||
@{nodes}= Create List alpha beta
|
||||
Create Linear Graph With Non Functional Nodes ${nodes}
|
||||
Set Empty Registry
|
||||
${result}= Execute Graph 7
|
||||
Should Be Equal As Integers ${result} 7
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Robot Framework library for PureGraph integration testing."""
|
||||
|
||||
import contextlib
|
||||
|
||||
|
||||
class PureGraphLibrary:
|
||||
"""Custom keywords for exercising PureGraph functionality."""
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = True
|
||||
|
||||
def __init__(self):
|
||||
self._graph = None
|
||||
self._fn_registry = {}
|
||||
self._topo_order = None
|
||||
self._exec_result = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Graph construction helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def create_linear_graph_with_function_nodes(self, node_names, function_names):
|
||||
"""Create a linear graph with the specified node and function names."""
|
||||
from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.pure_graph import PureGraph
|
||||
|
||||
nodes = {
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
}
|
||||
edges = []
|
||||
previous = "start"
|
||||
for node_name, function_name in zip(node_names, function_names, strict=False):
|
||||
nodes[node_name] = NodeConfig(
|
||||
name=node_name, type=NodeType.FUNCTION, function=function_name
|
||||
)
|
||||
edges.append(Edge(source=previous, target=node_name))
|
||||
previous = node_name
|
||||
edges.append(Edge(source=previous, target="end"))
|
||||
|
||||
self._graph = PureGraph(name="robot-test-graph", nodes=nodes, edges=edges)
|
||||
|
||||
def create_linear_graph_with_non_functional_nodes(self, node_names):
|
||||
"""Create a linear graph with non-functional (inert) nodes."""
|
||||
from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.pure_graph import PureGraph
|
||||
|
||||
nodes = {
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
}
|
||||
edges = []
|
||||
previous = "start"
|
||||
for node_name in node_names:
|
||||
nodes[node_name] = NodeConfig(
|
||||
name=node_name, type=NodeType.FUNCTION, function=None
|
||||
)
|
||||
edges.append(Edge(source=previous, target=node_name))
|
||||
previous = node_name
|
||||
edges.append(Edge(source=previous, target="end"))
|
||||
|
||||
self._graph = PureGraph(name="robot-test-inert", nodes=nodes, edges=edges)
|
||||
|
||||
def create_graph_with_missing_function(self, unregistered_fn_name):
|
||||
"""Create a graph where a node references a function not in the registry."""
|
||||
from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.pure_graph import PureGraph
|
||||
|
||||
nodes = {
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
}
|
||||
edges = []
|
||||
previous = "start"
|
||||
node_name = f"skipper_{unregistered_fn_name}"
|
||||
nodes[node_name] = NodeConfig(
|
||||
name=node_name, type=NodeType.FUNCTION, function=unregistered_fn_name
|
||||
)
|
||||
edges.append(Edge(source=previous, target=node_name))
|
||||
previous = node_name
|
||||
edges.append(Edge(source=previous, target="end"))
|
||||
|
||||
self._graph = PureGraph(name="robot-test-missing-fn", nodes=nodes, edges=edges)
|
||||
self._fn_registry = {}
|
||||
|
||||
def set_function_registry(self, registry_dict):
|
||||
|
HAL9001
commented
BLOCKING —
Project rules also require all imports to be at the top of the file — this placement is a secondary violation. HOW to fix: Remove Automated by CleverAgents Bot **BLOCKING — `import ast` is Unused — Causes `lint` CI Failure (F401)**
`import ast` is declared inside this method but `ast` is never referenced anywhere in `set_function_registry()`. Only bare `eval()` is called directly. `ruff check robot/` flags this as F401 and fails the `lint` CI job.
Project rules also require all imports to be at the top of the file — this placement is a secondary violation.
**HOW to fix:** Remove `import ast` from line 85 entirely. If you intended `ast.literal_eval()` (safer than `eval`), move `import ast` to the top of the file and replace `eval(value)` with `ast.literal_eval(value)`.
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
|
||||
"""Set or update the function registry used by execute()."""
|
||||
self._fn_registry = {}
|
||||
for key, value in registry_dict.items():
|
||||
if callable(value):
|
||||
self._fn_registry[key] = value
|
||||
elif isinstance(value, str):
|
||||
with contextlib.suppress(Exception):
|
||||
self._fn_registry[key] = eval(value)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Topological order queries
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def compute_topological_order(self):
|
||||
"""Compute and return the topological ordering of the current graph."""
|
||||
self._topo_order = self._graph.topological_order()
|
||||
|
||||
def get_topological_order(self):
|
||||
"""Return the last computed topological order as a list."""
|
||||
return self._topo_order or []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execution queries
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def execute_graph(self, initial_value):
|
||||
"""Execute the current graph starting with the given initial value."""
|
||||
self._exec_result = self._graph.execute(
|
||||
|
HAL9001
commented
BLOCKING — Line Too Long (89 chars) — Causes This line is 89 characters, exceeding the project's 88-character ruff E501 limit. HOW to fix: Automated by CleverAgents Bot **BLOCKING — Line Too Long (89 chars) — Causes `lint` CI Failure (E501)**
This line is 89 characters, exceeding the project's 88-character ruff E501 limit. `ruff check robot/` will flag it and fail the `lint` CI job.
**HOW to fix:**
```python
self._exec_result = self._graph.execute(
self._fn_registry, initial=initial_value
)
```
---
Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker
|
||||
self._fn_registry, initial=initial_value
|
||||
)
|
||||
return self._exec_result
|
||||
|
||||
def get_execution_result(self):
|
||||
"""Return the last execution result."""
|
||||
return self._exec_result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Assertion helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def assert_topo_order_equals(self, *expected_nodes):
|
||||
"""Assert topological order matches expected: [start, nodes..., end]."""
|
||||
actual = self._topo_order or []
|
||||
expected = ["start", *expected_nodes, "end"]
|
||||
if actual != expected:
|
||||
raise AssertionError(
|
||||
f"Topological order mismatch: expected {expected}, got {actual}"
|
||||
)
|
||||
|
||||
def assert_execution_result_equals(self, expected):
|
||||
"""Assert the last execution result equals expected value."""
|
||||
actual = self._exec_result
|
||||
if actual != expected:
|
||||
raise AssertionError(f"Expected {expected}, got {actual}")
|
||||
BLOCKING — Trailing whitespace on this blank line.
Lines 25, 37, 39, and 44 contain 8 spaces instead of being truly empty.
ruff format --checkrejects files with trailing whitespace, which is why thelintCI job is failing.HOW to fix: run
ruff format benchmarks/pure_graph_bench.pyfrom the repo root — it will strip the trailing whitespace automatically. Commit and push the result.Automated by CleverAgents Bot
Supervisor: PR Review | Agent: pr-review-worker