test(benchmarks): add ASV benchmarks for LangGraph and TUI
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
"""ASV benchmarks for LangGraph execution hotspots.
|
||||
|
||||
Measures performance of:
|
||||
- GraphExecutor.step execution
|
||||
- RouterNode.route decision making
|
||||
- Checkpoint read/write operations
|
||||
- Graph state transitions
|
||||
- Node execution throughput
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from cleveragents.langgraph.graph import GraphConfig, LangGraph
|
||||
from cleveragents.langgraph.nodes import Edge, Node, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.state import GraphState
|
||||
except ModuleNotFoundError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
from cleveragents.langgraph.graph import GraphConfig, LangGraph
|
||||
from cleveragents.langgraph.nodes import Edge, Node, NodeConfig, NodeType
|
||||
from cleveragents.langgraph.state import GraphState
|
||||
|
||||
|
||||
def _create_simple_graph() -> LangGraph:
|
||||
"""Create a simple representative plan graph for benchmarking."""
|
||||
config = GraphConfig(
|
||||
name="bench-graph",
|
||||
nodes={
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"process": NodeConfig(name="process", type=NodeType.FUNCTION),
|
||||
"route": NodeConfig(name="route", type=NodeType.CONDITIONAL),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
},
|
||||
edges=[
|
||||
Edge(source="start", target="process"),
|
||||
Edge(source="process", target="route"),
|
||||
Edge(source="route", target="end"),
|
||||
],
|
||||
entry_point="start",
|
||||
checkpointing=False,
|
||||
)
|
||||
return LangGraph(config)
|
||||
|
||||
|
||||
def _create_checkpoint_graph() -> LangGraph:
|
||||
"""Create a graph with checkpointing enabled for persistence benchmarks."""
|
||||
checkpoint_dir = Path("/tmp/langgraph_bench_checkpoints")
|
||||
checkpoint_dir.mkdir(exist_ok=True)
|
||||
|
||||
config = GraphConfig(
|
||||
name="bench-graph-checkpoint",
|
||||
nodes={
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"process": NodeConfig(name="process", type=NodeType.FUNCTION),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
},
|
||||
edges=[
|
||||
Edge(source="start", target="process"),
|
||||
Edge(source="process", target="end"),
|
||||
],
|
||||
entry_point="start",
|
||||
checkpointing=True,
|
||||
checkpoint_dir=checkpoint_dir,
|
||||
)
|
||||
return LangGraph(config)
|
||||
|
||||
|
||||
def _create_complex_graph() -> LangGraph:
|
||||
"""Create a more complex graph with multiple nodes for throughput testing."""
|
||||
config = GraphConfig(
|
||||
name="bench-graph-complex",
|
||||
nodes={
|
||||
"start": NodeConfig(name="start", type=NodeType.START),
|
||||
"node1": NodeConfig(name="node1", type=NodeType.FUNCTION),
|
||||
"node2": NodeConfig(name="node2", type=NodeType.FUNCTION),
|
||||
"node3": NodeConfig(name="node3", type=NodeType.FUNCTION),
|
||||
"route": NodeConfig(name="route", type=NodeType.CONDITIONAL),
|
||||
"end": NodeConfig(name="end", type=NodeType.END),
|
||||
},
|
||||
edges=[
|
||||
Edge(source="start", target="node1"),
|
||||
Edge(source="node1", target="node2"),
|
||||
Edge(source="node2", target="node3"),
|
||||
Edge(source="node3", target="route"),
|
||||
Edge(source="route", target="end"),
|
||||
],
|
||||
entry_point="start",
|
||||
checkpointing=False,
|
||||
parallel_execution=True,
|
||||
)
|
||||
return LangGraph(config)
|
||||
|
||||
|
||||
class LangGraphExecutionSuite:
|
||||
"""Benchmark LangGraph execution hotspots."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize graphs for benchmarking."""
|
||||
self.simple_graph = _create_simple_graph()
|
||||
self.checkpoint_graph = _create_checkpoint_graph()
|
||||
self.complex_graph = _create_complex_graph()
|
||||
|
||||
# Sample input state
|
||||
self.sample_state = GraphState()
|
||||
|
||||
def time_simple_graph_execute(self) -> None:
|
||||
"""Measure simple graph execution time."""
|
||||
asyncio.run(self.simple_graph.execute(self.sample_state))
|
||||
|
||||
def time_complex_graph_execute(self) -> None:
|
||||
"""Measure complex graph execution time."""
|
||||
asyncio.run(self.complex_graph.execute(self.sample_state))
|
||||
|
||||
def time_checkpoint_graph_execute(self) -> None:
|
||||
"""Measure graph execution with checkpointing enabled."""
|
||||
asyncio.run(self.checkpoint_graph.execute(self.sample_state))
|
||||
|
||||
def time_state_manager_get_state(self) -> None:
|
||||
"""Measure state manager state retrieval."""
|
||||
self.simple_graph.state_manager.get_state()
|
||||
|
||||
def time_state_manager_update_state(self) -> None:
|
||||
"""Measure state manager state update."""
|
||||
new_state = GraphState()
|
||||
self.simple_graph.state_manager.state = new_state
|
||||
|
||||
def track_graph_nodes_count(self) -> int:
|
||||
"""Track number of nodes in simple graph."""
|
||||
return len(self.simple_graph.nodes)
|
||||
|
||||
def track_graph_edges_count(self) -> int:
|
||||
"""Track number of edges in simple graph."""
|
||||
return len(self.simple_graph.config.edges)
|
||||
|
||||
def track_execution_history_length(self) -> int:
|
||||
"""Track execution history length after execution."""
|
||||
return len(self.simple_graph.get_execution_history())
|
||||
|
||||
|
||||
class LangGraphRouterNodeSuite:
|
||||
"""Benchmark RouterNode routing performance."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize router node for benchmarking."""
|
||||
self.router_config = NodeConfig(
|
||||
name="router",
|
||||
type=NodeType.CONDITIONAL,
|
||||
condition={"type": "simple_condition"},
|
||||
)
|
||||
self.router_node = Node(self.router_config)
|
||||
|
||||
def time_router_node_creation(self) -> None:
|
||||
"""Measure router node instantiation time."""
|
||||
Node(self.router_config)
|
||||
|
||||
def time_router_node_config_access(self) -> None:
|
||||
"""Measure router node config access."""
|
||||
_ = self.router_node.config.condition
|
||||
|
||||
|
||||
class LangGraphStateTransitionSuite:
|
||||
"""Benchmark graph state transitions and updates."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize state for benchmarking."""
|
||||
self.state = GraphState()
|
||||
self.graph = _create_simple_graph()
|
||||
|
||||
def time_state_creation(self) -> None:
|
||||
"""Measure GraphState creation time."""
|
||||
GraphState()
|
||||
|
||||
def time_state_dict_conversion(self) -> None:
|
||||
"""Measure state to dict conversion."""
|
||||
self.state.to_dict()
|
||||
|
||||
def time_state_from_dict(self) -> None:
|
||||
"""Measure state from dict creation."""
|
||||
GraphState.from_dict({"messages": []})
|
||||
|
||||
def time_state_copy(self) -> None:
|
||||
"""Measure state copy operation."""
|
||||
self.state.copy()
|
||||
@@ -0,0 +1,200 @@
|
||||
"""ASV benchmarks for TUI rendering and command routing performance.
|
||||
|
||||
Measures performance of:
|
||||
- TUI layout and render cycle time
|
||||
- Slash command catalog hydration
|
||||
- Command routing overhead
|
||||
- Widget rendering performance
|
||||
- Session view updates
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from cleveragents.tui.slash_catalog import (
|
||||
SLASH_COMMAND_SPECS,
|
||||
SlashCommandSpec,
|
||||
slash_command_specs,
|
||||
)
|
||||
except ModuleNotFoundError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
from cleveragents.tui.slash_catalog import (
|
||||
SLASH_COMMAND_SPECS,
|
||||
SlashCommandSpec,
|
||||
slash_command_specs,
|
||||
)
|
||||
|
||||
|
||||
def _build_command_index() -> dict[str, list[SlashCommandSpec]]:
|
||||
"""Build a command index grouped by category."""
|
||||
index: dict[str, list[SlashCommandSpec]] = {}
|
||||
for spec in SLASH_COMMAND_SPECS:
|
||||
if spec.group not in index:
|
||||
index[spec.group] = []
|
||||
index[spec.group].append(spec)
|
||||
return index
|
||||
|
||||
|
||||
def _filter_commands_by_prefix(prefix: str) -> list[SlashCommandSpec]:
|
||||
"""Filter commands by prefix for autocomplete."""
|
||||
return [cmd for cmd in SLASH_COMMAND_SPECS if cmd.command.startswith(prefix)]
|
||||
|
||||
|
||||
class TUISlashCatalogSuite:
|
||||
"""Benchmark TUI slash command catalog operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize catalog for benchmarking."""
|
||||
self.catalog = SLASH_COMMAND_SPECS
|
||||
self.command_index = _build_command_index()
|
||||
|
||||
def time_catalog_iteration(self) -> None:
|
||||
"""Measure time to iterate through all commands."""
|
||||
for _ in self.catalog:
|
||||
pass
|
||||
|
||||
def time_catalog_grouping(self) -> None:
|
||||
"""Measure time to group commands by category."""
|
||||
_build_command_index()
|
||||
|
||||
def time_catalog_prefix_filter(self) -> None:
|
||||
"""Measure time to filter commands by prefix."""
|
||||
_filter_commands_by_prefix("session:")
|
||||
|
||||
def time_catalog_search(self) -> None:
|
||||
"""Measure time to search for a specific command."""
|
||||
target = "plan:execute"
|
||||
for cmd in self.catalog:
|
||||
if cmd.command == target:
|
||||
break
|
||||
|
||||
def time_command_spec_creation(self) -> None:
|
||||
"""Measure time to create a SlashCommandSpec."""
|
||||
SlashCommandSpec(
|
||||
command="test:command",
|
||||
group="Test",
|
||||
description="Test command",
|
||||
)
|
||||
|
||||
def track_catalog_size(self) -> int:
|
||||
"""Track total number of commands in catalog."""
|
||||
return len(self.catalog)
|
||||
|
||||
def track_unique_groups(self) -> int:
|
||||
"""Track number of unique command groups."""
|
||||
return len(self.command_index)
|
||||
|
||||
def track_largest_group_size(self) -> int:
|
||||
"""Track size of largest command group."""
|
||||
return max(len(cmds) for cmds in self.command_index.values())
|
||||
|
||||
|
||||
class TUICommandRoutingSuite:
|
||||
"""Benchmark TUI command routing performance."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize routing structures for benchmarking."""
|
||||
self.catalog = SLASH_COMMAND_SPECS
|
||||
self.command_map = {cmd.command: cmd for cmd in self.catalog}
|
||||
|
||||
def time_command_lookup(self) -> None:
|
||||
"""Measure time to lookup a command in the map."""
|
||||
_ = self.command_map.get("session:create")
|
||||
|
||||
def time_command_validation(self) -> None:
|
||||
"""Measure time to validate a command exists."""
|
||||
command = "plan:execute"
|
||||
command in self.command_map
|
||||
|
||||
def time_group_lookup(self) -> None:
|
||||
"""Measure time to find all commands in a group."""
|
||||
group = "Session"
|
||||
[cmd for cmd in self.catalog if cmd.group == group]
|
||||
|
||||
def time_description_search(self) -> None:
|
||||
"""Measure time to search commands by description."""
|
||||
search_term = "session"
|
||||
[
|
||||
cmd
|
||||
for cmd in self.catalog
|
||||
if search_term.lower() in cmd.description.lower()
|
||||
]
|
||||
|
||||
def track_command_map_size(self) -> int:
|
||||
"""Track size of command lookup map."""
|
||||
return len(self.command_map)
|
||||
|
||||
|
||||
class TUISessionViewSuite:
|
||||
"""Benchmark TUI session view operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize session view for benchmarking."""
|
||||
from cleveragents.tui.app import SessionView
|
||||
|
||||
self.session = SessionView(
|
||||
session_id="bench-session-001",
|
||||
transcript=["message 1", "message 2", "message 3"],
|
||||
)
|
||||
|
||||
def time_session_view_creation(self) -> None:
|
||||
"""Measure time to create a SessionView."""
|
||||
from cleveragents.tui.app import SessionView
|
||||
|
||||
SessionView(
|
||||
session_id="test-session",
|
||||
transcript=["test message"],
|
||||
)
|
||||
|
||||
def time_transcript_append(self) -> None:
|
||||
"""Measure time to append to transcript."""
|
||||
self.session.transcript.append("new message")
|
||||
|
||||
def time_transcript_iteration(self) -> None:
|
||||
"""Measure time to iterate through transcript."""
|
||||
for _ in self.session.transcript:
|
||||
pass
|
||||
|
||||
def track_session_id_length(self) -> int:
|
||||
"""Track session ID length."""
|
||||
return len(self.session.session_id)
|
||||
|
||||
def track_transcript_size(self) -> int:
|
||||
"""Track transcript message count."""
|
||||
return len(self.session.transcript)
|
||||
|
||||
|
||||
class TUIWidgetRenderingSuite:
|
||||
"""Benchmark TUI widget rendering operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Initialize widgets for benchmarking."""
|
||||
self.command_specs = SLASH_COMMAND_SPECS
|
||||
self.sample_text = "Sample widget content for rendering"
|
||||
|
||||
def time_command_spec_string_conversion(self) -> None:
|
||||
"""Measure time to convert command spec to string."""
|
||||
spec = self.command_specs[0]
|
||||
str(spec)
|
||||
|
||||
def time_command_spec_formatting(self) -> None:
|
||||
"""Measure time to format command spec for display."""
|
||||
spec = self.command_specs[0]
|
||||
f"{spec.command} - {spec.description}"
|
||||
|
||||
def time_text_rendering_short(self) -> None:
|
||||
"""Measure time to render short text."""
|
||||
len(self.sample_text)
|
||||
|
||||
def time_text_rendering_long(self) -> None:
|
||||
"""Measure time to render long text."""
|
||||
long_text = self.sample_text * 100
|
||||
len(long_text)
|
||||
|
||||
def track_widget_content_size(self) -> int:
|
||||
"""Track size of widget content."""
|
||||
return len(self.sample_text)
|
||||
Reference in New Issue
Block a user