diff --git a/CHANGELOG.md b/CHANGELOG.md index 3975b63ea..898abc7a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -903,6 +903,21 @@ uko-oo:Class` triple emission in `PythonAnalyzer._extract_class()` so that ``max_context_files`` (default: 5). Non-positive values raise ``ValueError``. All existing call sites remain backward-compatible via default arguments. +### Tests + +- **PureGraph BDD and Integration Test Coverage** (#9601): Added comprehensive test + coverage for the PureGraph module, which previously had orphaned Behave step definitions + (`features/steps/pure_graph_coverage_steps.py`) with no driving scenarios. The PureGraph + scenarios (topological ordering, function execution with dependency resolution, missing + function fallback behavior, and inert non-functional node handling) are wired through + `features/consolidated_langgraph.feature` to reuse the existing step definitions without + introducing a duplicate standalone feature file. Introduces Robot Framework integration + tests in `robot/langgraph/pure_graph.robot` (backed by the + `robot/langgraph/pure_graph_lib.py` Python library) exercising the PureGraph workflow + end-to-end. Includes ASV benchmarks in `benchmarks/pure_graph_bench.py` measuring + execution throughput and topological ordering performance across increasing node counts + (10, 50, 100, 500). + ### Added - **ACMS Index Data Model and File Traversal Engine** (#9579): Implements the diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 5dbad62b8..c3e094d08 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -33,6 +33,7 @@ Below are some of the specific details of various contributions. * HAMZA KHYARI has contributed the ACMS execute-phase context assembler project-level hot_max_tokens fix (PR #11036 / issue #11035): added `_resolve_effective_budget()` method that reads each linked project's `settings.hot_max_tokens` and uses the maximum override value as the pipeline budget instead of the hardcoded global 16K default. * HAL 9000 has contributed the automated CLI docstring example validation (#9106): added `DocstringExampleValidator` to enforce positional-before-option ordering in CLI `Examples:` sections, with Behave test coverage and CONTRIBUTING.md documentation. * HAL 9000 has contributed the AutoDebugAgent prompt injection mitigation fix (#9110): sanitized user-provided `error_message` and `code_context` fields in all three agent methods using `PromptSanitizer` boundary markers, added graceful `PromptInjectionDetected` exception handling, and added BDD and Robot Framework integration tests for the security fix. +* HAL 9000 has contributed the PureGraph BDD coverage suite (PR #9601 / issue #9531): wired the previously orphaned `features/steps/pure_graph_coverage_steps.py` definitions through the existing `features/consolidated_langgraph.feature` (topological ordering, function execution, missing function fallback, and non-functional node handling); created Robot Framework integration tests in `robot/langgraph/pure_graph.robot` backed by the `robot/langgraph/pure_graph_lib.py` Python library; and implemented ASV benchmarks in `benchmarks/pure_graph_bench.py` measuring execution throughput across varying node counts. * This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc. * HAL 9000 has contributed automated bug fixes, CLI output formatting improvements, and ongoing maintenance as part of the CleverAgents automation system. * HAL 9000 has contributed the pr-review-pool-supervisor tracking prefix documentation fix (#7891): aligned all documentation references from the outdated `AUTO-REV-POOL` prefix to the correct `AUTO-REV-SUP` prefix used in production. diff --git a/benchmarks/pure_graph_bench.py b/benchmarks/pure_graph_bench.py new file mode 100644 index 000000000..de376735e --- /dev/null +++ b/benchmarks/pure_graph_bench.py @@ -0,0 +1,58 @@ +"""Benchmarks for PureGraph execution throughput and ordering.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType +from cleveragents.langgraph.pure_graph import PureGraph + + +class PureGraphBench: + """Benchmark suite for PureGraph execution performance.""" + + params: ClassVar[list[int]] = [10, 50, 100, 500] + param_names: ClassVar[list[str]] = ["node_count"] + + def setup(self, node_count: int) -> None: + """Set up benchmark with varying node counts.""" + self.node_count = node_count + self.nodes: dict[str, NodeConfig] = { + "start": NodeConfig(name="start", type=NodeType.START), + "end": NodeConfig(name="end", type=NodeType.END), + } + self.edges: list[Edge] = [] + + # Create a linear chain of nodes + previous = "start" + for i in range(node_count): + node_name = f"node_{i}" + self.nodes[node_name] = NodeConfig( + name=node_name, type=NodeType.FUNCTION, function=f"fn_{i}" + ) + self.edges.append(Edge(source=previous, target=node_name)) + previous = node_name + + self.edges.append(Edge(source=previous, target="end")) + + # Create function registry with identity functions + self.fn_registry: dict[str, Any] = { + f"fn_{i}": lambda x, i=i: x for i in range(node_count) + } + + self.graph = PureGraph( + name=f"bench_graph_{node_count}", nodes=self.nodes, edges=self.edges + ) + + def time_topological_order(self, node_count: int) -> None: + """Benchmark topological ordering.""" + self.graph.topological_order() + + def time_execute(self, node_count: int) -> None: + """Benchmark graph execution.""" + self.graph.execute(self.fn_registry, initial=0) + + def time_execute_with_result(self, node_count: int) -> None: + """Benchmark graph execution with result accumulation.""" + result = self.graph.execute(self.fn_registry, initial=42) + assert result == 42, f"Expected 42, got {result}" diff --git a/robot/langgraph/pure_graph.robot b/robot/langgraph/pure_graph.robot new file mode 100644 index 000000000..c9b004a6e --- /dev/null +++ b/robot/langgraph/pure_graph.robot @@ -0,0 +1,49 @@ +*** Settings *** +Documentation Integration tests for PureGraph execution and ordering +Library Collections +Library ${CURDIR}/pure_graph_lib.py + +*** Keywords *** +Set Double And Increment Registry + ${double}= Evaluate lambda x: x * 2 + ${increment}= Evaluate lambda x: x + 1 + ${registry}= Create Dictionary double=${double} increment=${increment} + Set Function Registry ${registry} + +Set Empty Registry + ${empty}= Create Dictionary + Set Function Registry ${empty} + +*** Test Cases *** +PureGraph Topological Order + [Documentation] Verify topological ordering of graph nodes respects start/end boundaries + @{nodes}= Create List alpha beta + Create Linear Graph With Non Functional Nodes ${nodes} + Compute Topological Order + ${topo}= Get Topological Order + @{expected}= Create List start alpha beta end + Lists Should Be Equal ${expected} ${topo} + +PureGraph Execute With Functions + [Documentation] Verify function execution applies transformations sequentially in declaration order + @{node_names}= Create List double increment + @{func_names}= Create List double increment + Create Linear Graph With Function Nodes ${node_names} ${func_names} + Set Double And Increment Registry + ${result}= Execute Graph ${1} + Should Be Equal As Integers ${result} 3 + +PureGraph Execute With Missing Function + [Documentation] Verify missing function nodes are skipped gracefully without error + Create Graph With Missing Function missing_fn + Set Empty Registry + ${result}= Execute Graph 5 + Should Be Equal As Integers ${result} 5 + +PureGraph Execute With Non-Functional Nodes + [Documentation] Verify non-functional (inert) nodes do not change the value + @{nodes}= Create List alpha beta + Create Linear Graph With Non Functional Nodes ${nodes} + Set Empty Registry + ${result}= Execute Graph 7 + Should Be Equal As Integers ${result} 7 diff --git a/robot/langgraph/pure_graph_lib.py b/robot/langgraph/pure_graph_lib.py new file mode 100644 index 000000000..539fc05e3 --- /dev/null +++ b/robot/langgraph/pure_graph_lib.py @@ -0,0 +1,139 @@ +"""Robot Framework library for PureGraph integration testing.""" + +import contextlib + + +class PureGraphLibrary: + """Custom keywords for exercising PureGraph functionality.""" + + ROBOT_AUTO_KEYWORDS = True + + def __init__(self): + self._graph = None + self._fn_registry = {} + self._topo_order = None + self._exec_result = None + + # ------------------------------------------------------------------ + # Graph construction helpers + # ------------------------------------------------------------------ + + def create_linear_graph_with_function_nodes(self, node_names, function_names): + """Create a linear graph with the specified node and function names.""" + from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType + from cleveragents.langgraph.pure_graph import PureGraph + + nodes = { + "start": NodeConfig(name="start", type=NodeType.START), + "end": NodeConfig(name="end", type=NodeType.END), + } + edges = [] + previous = "start" + for node_name, function_name in zip(node_names, function_names, strict=False): + nodes[node_name] = NodeConfig( + name=node_name, type=NodeType.FUNCTION, function=function_name + ) + edges.append(Edge(source=previous, target=node_name)) + previous = node_name + edges.append(Edge(source=previous, target="end")) + + self._graph = PureGraph(name="robot-test-graph", nodes=nodes, edges=edges) + + def create_linear_graph_with_non_functional_nodes(self, node_names): + """Create a linear graph with non-functional (inert) nodes.""" + from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType + from cleveragents.langgraph.pure_graph import PureGraph + + nodes = { + "start": NodeConfig(name="start", type=NodeType.START), + "end": NodeConfig(name="end", type=NodeType.END), + } + edges = [] + previous = "start" + for node_name in node_names: + nodes[node_name] = NodeConfig( + name=node_name, type=NodeType.FUNCTION, function=None + ) + edges.append(Edge(source=previous, target=node_name)) + previous = node_name + edges.append(Edge(source=previous, target="end")) + + self._graph = PureGraph(name="robot-test-inert", nodes=nodes, edges=edges) + + def create_graph_with_missing_function(self, unregistered_fn_name): + """Create a graph where a node references a function not in the registry.""" + from cleveragents.langgraph.nodes import Edge, NodeConfig, NodeType + from cleveragents.langgraph.pure_graph import PureGraph + + nodes = { + "start": NodeConfig(name="start", type=NodeType.START), + "end": NodeConfig(name="end", type=NodeType.END), + } + edges = [] + previous = "start" + node_name = f"skipper_{unregistered_fn_name}" + nodes[node_name] = NodeConfig( + name=node_name, type=NodeType.FUNCTION, function=unregistered_fn_name + ) + edges.append(Edge(source=previous, target=node_name)) + previous = node_name + edges.append(Edge(source=previous, target="end")) + + self._graph = PureGraph(name="robot-test-missing-fn", nodes=nodes, edges=edges) + self._fn_registry = {} + + def set_function_registry(self, registry_dict): + """Set or update the function registry used by execute().""" + self._fn_registry = {} + for key, value in registry_dict.items(): + if callable(value): + self._fn_registry[key] = value + elif isinstance(value, str): + with contextlib.suppress(Exception): + self._fn_registry[key] = eval(value) + + # ------------------------------------------------------------------ + # Topological order queries + # ------------------------------------------------------------------ + + def compute_topological_order(self): + """Compute and return the topological ordering of the current graph.""" + self._topo_order = self._graph.topological_order() + + def get_topological_order(self): + """Return the last computed topological order as a list.""" + return self._topo_order or [] + + # ------------------------------------------------------------------ + # Execution queries + # ------------------------------------------------------------------ + + def execute_graph(self, initial_value): + """Execute the current graph starting with the given initial value.""" + self._exec_result = self._graph.execute( + self._fn_registry, initial=initial_value + ) + return self._exec_result + + def get_execution_result(self): + """Return the last execution result.""" + return self._exec_result + + # ------------------------------------------------------------------ + # Assertion helpers + # ------------------------------------------------------------------ + + def assert_topo_order_equals(self, *expected_nodes): + """Assert topological order matches expected: [start, nodes..., end].""" + actual = self._topo_order or [] + expected = ["start", *expected_nodes, "end"] + if actual != expected: + raise AssertionError( + f"Topological order mismatch: expected {expected}, got {actual}" + ) + + def assert_execution_result_equals(self, expected): + """Assert the last execution result equals expected value.""" + actual = self._exec_result + if actual != expected: + raise AssertionError(f"Expected {expected}, got {actual}")