diff --git a/benchmarks/langgraph_execution_bench.py b/benchmarks/langgraph_execution_bench.py new file mode 100644 index 000000000..97833b1f9 --- /dev/null +++ b/benchmarks/langgraph_execution_bench.py @@ -0,0 +1,188 @@ +"""ASV benchmarks for LangGraph execution hotspots. + +Measures performance of: +- GraphExecutor.step execution +- RouterNode.route decision making +- Checkpoint read/write operations +- Graph state transitions +- Node execution throughput +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any + +try: + from cleveragents.langgraph.graph import GraphConfig, LangGraph + from cleveragents.langgraph.nodes import Edge, Node, NodeConfig, NodeType + from cleveragents.langgraph.state import GraphState +except ModuleNotFoundError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cleveragents.langgraph.graph import GraphConfig, LangGraph + from cleveragents.langgraph.nodes import Edge, Node, NodeConfig, NodeType + from cleveragents.langgraph.state import GraphState + + +def _create_simple_graph() -> LangGraph: + """Create a simple representative plan graph for benchmarking.""" + config = GraphConfig( + name="bench-graph", + nodes={ + "start": NodeConfig(name="start", type=NodeType.START), + "process": NodeConfig(name="process", type=NodeType.FUNCTION), + "route": NodeConfig(name="route", type=NodeType.CONDITIONAL), + "end": NodeConfig(name="end", type=NodeType.END), + }, + edges=[ + Edge(source="start", target="process"), + Edge(source="process", target="route"), + Edge(source="route", target="end"), + ], + entry_point="start", + checkpointing=False, + ) + return LangGraph(config) + + +def _create_checkpoint_graph() -> LangGraph: + """Create a graph with checkpointing enabled for persistence benchmarks.""" + checkpoint_dir = Path("/tmp/langgraph_bench_checkpoints") + checkpoint_dir.mkdir(exist_ok=True) + + config = GraphConfig( + name="bench-graph-checkpoint", + nodes={ + "start": NodeConfig(name="start", type=NodeType.START), + "process": NodeConfig(name="process", type=NodeType.FUNCTION), + "end": NodeConfig(name="end", type=NodeType.END), + }, + edges=[ + Edge(source="start", target="process"), + Edge(source="process", target="end"), + ], + entry_point="start", + checkpointing=True, + checkpoint_dir=checkpoint_dir, + ) + return LangGraph(config) + + +def _create_complex_graph() -> LangGraph: + """Create a more complex graph with multiple nodes for throughput testing.""" + config = GraphConfig( + name="bench-graph-complex", + nodes={ + "start": NodeConfig(name="start", type=NodeType.START), + "node1": NodeConfig(name="node1", type=NodeType.FUNCTION), + "node2": NodeConfig(name="node2", type=NodeType.FUNCTION), + "node3": NodeConfig(name="node3", type=NodeType.FUNCTION), + "route": NodeConfig(name="route", type=NodeType.CONDITIONAL), + "end": NodeConfig(name="end", type=NodeType.END), + }, + edges=[ + Edge(source="start", target="node1"), + Edge(source="node1", target="node2"), + Edge(source="node2", target="node3"), + Edge(source="node3", target="route"), + Edge(source="route", target="end"), + ], + entry_point="start", + checkpointing=False, + parallel_execution=True, + ) + return LangGraph(config) + + +class LangGraphExecutionSuite: + """Benchmark LangGraph execution hotspots.""" + + def setup(self) -> None: + """Initialize graphs for benchmarking.""" + self.simple_graph = _create_simple_graph() + self.checkpoint_graph = _create_checkpoint_graph() + self.complex_graph = _create_complex_graph() + + # Sample input state + self.sample_state = GraphState() + + def time_simple_graph_execute(self) -> None: + """Measure simple graph execution time.""" + asyncio.run(self.simple_graph.execute(self.sample_state)) + + def time_complex_graph_execute(self) -> None: + """Measure complex graph execution time.""" + asyncio.run(self.complex_graph.execute(self.sample_state)) + + def time_checkpoint_graph_execute(self) -> None: + """Measure graph execution with checkpointing enabled.""" + asyncio.run(self.checkpoint_graph.execute(self.sample_state)) + + def time_state_manager_get_state(self) -> None: + """Measure state manager state retrieval.""" + self.simple_graph.state_manager.get_state() + + def time_state_manager_update_state(self) -> None: + """Measure state manager state update.""" + new_state = GraphState() + self.simple_graph.state_manager.state = new_state + + def track_graph_nodes_count(self) -> int: + """Track number of nodes in simple graph.""" + return len(self.simple_graph.nodes) + + def track_graph_edges_count(self) -> int: + """Track number of edges in simple graph.""" + return len(self.simple_graph.config.edges) + + def track_execution_history_length(self) -> int: + """Track execution history length after execution.""" + return len(self.simple_graph.get_execution_history()) + + +class LangGraphRouterNodeSuite: + """Benchmark RouterNode routing performance.""" + + def setup(self) -> None: + """Initialize router node for benchmarking.""" + self.router_config = NodeConfig( + name="router", + type=NodeType.CONDITIONAL, + condition={"type": "simple_condition"}, + ) + self.router_node = Node(self.router_config) + + def time_router_node_creation(self) -> None: + """Measure router node instantiation time.""" + Node(self.router_config) + + def time_router_node_config_access(self) -> None: + """Measure router node config access.""" + _ = self.router_node.config.condition + + +class LangGraphStateTransitionSuite: + """Benchmark graph state transitions and updates.""" + + def setup(self) -> None: + """Initialize state for benchmarking.""" + self.state = GraphState() + self.graph = _create_simple_graph() + + def time_state_creation(self) -> None: + """Measure GraphState creation time.""" + GraphState() + + def time_state_dict_conversion(self) -> None: + """Measure state to dict conversion.""" + self.state.to_dict() + + def time_state_from_dict(self) -> None: + """Measure state from dict creation.""" + GraphState.from_dict({"messages": []}) + + def time_state_copy(self) -> None: + """Measure state copy operation.""" + self.state.copy() diff --git a/benchmarks/tui_rendering_bench.py b/benchmarks/tui_rendering_bench.py new file mode 100644 index 000000000..413b663e1 --- /dev/null +++ b/benchmarks/tui_rendering_bench.py @@ -0,0 +1,200 @@ +"""ASV benchmarks for TUI rendering and command routing performance. + +Measures performance of: +- TUI layout and render cycle time +- Slash command catalog hydration +- Command routing overhead +- Widget rendering performance +- Session view updates +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +try: + from cleveragents.tui.slash_catalog import ( + SLASH_COMMAND_SPECS, + SlashCommandSpec, + slash_command_specs, + ) +except ModuleNotFoundError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cleveragents.tui.slash_catalog import ( + SLASH_COMMAND_SPECS, + SlashCommandSpec, + slash_command_specs, + ) + + +def _build_command_index() -> dict[str, list[SlashCommandSpec]]: + """Build a command index grouped by category.""" + index: dict[str, list[SlashCommandSpec]] = {} + for spec in SLASH_COMMAND_SPECS: + if spec.group not in index: + index[spec.group] = [] + index[spec.group].append(spec) + return index + + +def _filter_commands_by_prefix(prefix: str) -> list[SlashCommandSpec]: + """Filter commands by prefix for autocomplete.""" + return [cmd for cmd in SLASH_COMMAND_SPECS if cmd.command.startswith(prefix)] + + +class TUISlashCatalogSuite: + """Benchmark TUI slash command catalog operations.""" + + def setup(self) -> None: + """Initialize catalog for benchmarking.""" + self.catalog = SLASH_COMMAND_SPECS + self.command_index = _build_command_index() + + def time_catalog_iteration(self) -> None: + """Measure time to iterate through all commands.""" + for _ in self.catalog: + pass + + def time_catalog_grouping(self) -> None: + """Measure time to group commands by category.""" + _build_command_index() + + def time_catalog_prefix_filter(self) -> None: + """Measure time to filter commands by prefix.""" + _filter_commands_by_prefix("session:") + + def time_catalog_search(self) -> None: + """Measure time to search for a specific command.""" + target = "plan:execute" + for cmd in self.catalog: + if cmd.command == target: + break + + def time_command_spec_creation(self) -> None: + """Measure time to create a SlashCommandSpec.""" + SlashCommandSpec( + command="test:command", + group="Test", + description="Test command", + ) + + def track_catalog_size(self) -> int: + """Track total number of commands in catalog.""" + return len(self.catalog) + + def track_unique_groups(self) -> int: + """Track number of unique command groups.""" + return len(self.command_index) + + def track_largest_group_size(self) -> int: + """Track size of largest command group.""" + return max(len(cmds) for cmds in self.command_index.values()) + + +class TUICommandRoutingSuite: + """Benchmark TUI command routing performance.""" + + def setup(self) -> None: + """Initialize routing structures for benchmarking.""" + self.catalog = SLASH_COMMAND_SPECS + self.command_map = {cmd.command: cmd for cmd in self.catalog} + + def time_command_lookup(self) -> None: + """Measure time to lookup a command in the map.""" + _ = self.command_map.get("session:create") + + def time_command_validation(self) -> None: + """Measure time to validate a command exists.""" + command = "plan:execute" + command in self.command_map + + def time_group_lookup(self) -> None: + """Measure time to find all commands in a group.""" + group = "Session" + [cmd for cmd in self.catalog if cmd.group == group] + + def time_description_search(self) -> None: + """Measure time to search commands by description.""" + search_term = "session" + [ + cmd + for cmd in self.catalog + if search_term.lower() in cmd.description.lower() + ] + + def track_command_map_size(self) -> int: + """Track size of command lookup map.""" + return len(self.command_map) + + +class TUISessionViewSuite: + """Benchmark TUI session view operations.""" + + def setup(self) -> None: + """Initialize session view for benchmarking.""" + from cleveragents.tui.app import SessionView + + self.session = SessionView( + session_id="bench-session-001", + transcript=["message 1", "message 2", "message 3"], + ) + + def time_session_view_creation(self) -> None: + """Measure time to create a SessionView.""" + from cleveragents.tui.app import SessionView + + SessionView( + session_id="test-session", + transcript=["test message"], + ) + + def time_transcript_append(self) -> None: + """Measure time to append to transcript.""" + self.session.transcript.append("new message") + + def time_transcript_iteration(self) -> None: + """Measure time to iterate through transcript.""" + for _ in self.session.transcript: + pass + + def track_session_id_length(self) -> int: + """Track session ID length.""" + return len(self.session.session_id) + + def track_transcript_size(self) -> int: + """Track transcript message count.""" + return len(self.session.transcript) + + +class TUIWidgetRenderingSuite: + """Benchmark TUI widget rendering operations.""" + + def setup(self) -> None: + """Initialize widgets for benchmarking.""" + self.command_specs = SLASH_COMMAND_SPECS + self.sample_text = "Sample widget content for rendering" + + def time_command_spec_string_conversion(self) -> None: + """Measure time to convert command spec to string.""" + spec = self.command_specs[0] + str(spec) + + def time_command_spec_formatting(self) -> None: + """Measure time to format command spec for display.""" + spec = self.command_specs[0] + f"{spec.command} - {spec.description}" + + def time_text_rendering_short(self) -> None: + """Measure time to render short text.""" + len(self.sample_text) + + def time_text_rendering_long(self) -> None: + """Measure time to render long text.""" + long_text = self.sample_text * 100 + len(long_text) + + def track_widget_content_size(self) -> int: + """Track size of widget content.""" + return len(self.sample_text)