forked from cleveragents/cleveragents-core
132 lines
3.9 KiB
Python
132 lines
3.9 KiB
Python
"""ASV benchmarks for plan explain and plan tree operations.
|
|
|
|
Measures the performance of:
|
|
- Explain dict building with various flag combinations
|
|
- Decision tree construction and filtering
|
|
- Tree table rendering
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.cli.commands.plan import (
|
|
_build_explain_dict,
|
|
build_decision_tree,
|
|
)
|
|
from cleveragents.cli.formatting import format_output
|
|
from cleveragents.domain.models.core.decision import (
|
|
ContextSnapshot,
|
|
Decision,
|
|
DecisionType,
|
|
ResourceRef,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.cli.commands.plan import (
|
|
_build_explain_dict,
|
|
build_decision_tree,
|
|
)
|
|
from cleveragents.cli.formatting import format_output
|
|
from cleveragents.domain.models.core.decision import (
|
|
ContextSnapshot,
|
|
Decision,
|
|
DecisionType,
|
|
ResourceRef,
|
|
)
|
|
|
|
from ulid import ULID
|
|
|
|
_PLAN_ID = str(ULID())
|
|
|
|
|
|
def _make_decision(
|
|
seq: int,
|
|
parent_id: str | None = None,
|
|
decision_id: str | None = None,
|
|
dt: DecisionType = DecisionType.STRATEGY_CHOICE,
|
|
superseded_by: str | None = None,
|
|
) -> Decision:
|
|
"""Build a Decision for benchmarking."""
|
|
if dt == DecisionType.PROMPT_DEFINITION:
|
|
parent_id = None
|
|
elif parent_id is None:
|
|
parent_id = str(ULID())
|
|
return Decision(
|
|
decision_id=decision_id or str(ULID()),
|
|
plan_id=_PLAN_ID,
|
|
parent_decision_id=parent_id,
|
|
sequence_number=seq,
|
|
decision_type=dt,
|
|
question=f"Question {seq}?",
|
|
chosen_option=f"Option {seq}",
|
|
confidence_score=0.8,
|
|
superseded_by=superseded_by,
|
|
context_snapshot=ContextSnapshot(
|
|
hot_context_hash=f"sha256:bench{seq}",
|
|
relevant_resources=[
|
|
ResourceRef(resource_id=str(ULID()), path=f"src/f{seq}.py"),
|
|
],
|
|
),
|
|
rationale=f"Rationale for decision {seq}",
|
|
actor_reasoning=f"LLM reasoning for {seq}",
|
|
alternatives_considered=[f"alt-{seq}-a", f"alt-{seq}-b"],
|
|
)
|
|
|
|
|
|
class PlanExplainSuite:
|
|
"""Benchmark plan explain formatting."""
|
|
|
|
def setup(self) -> None:
|
|
self.decision = _make_decision(0, dt=DecisionType.PROMPT_DEFINITION)
|
|
|
|
def time_explain_formatting(self) -> None:
|
|
"""Benchmark basic explain dict building."""
|
|
_build_explain_dict(self.decision)
|
|
|
|
def time_explain_with_context(self) -> None:
|
|
"""Benchmark explain dict building with all flags."""
|
|
_build_explain_dict(
|
|
self.decision,
|
|
show_context=True,
|
|
show_reasoning=True,
|
|
show_alternatives=True,
|
|
)
|
|
|
|
|
|
class PlanTreeSuite:
|
|
"""Benchmark plan tree operations."""
|
|
|
|
def setup(self) -> None:
|
|
root_id = str(ULID())
|
|
self.root_id = root_id
|
|
self.decisions: list[Decision] = [
|
|
_make_decision(0, dt=DecisionType.PROMPT_DEFINITION, decision_id=root_id),
|
|
]
|
|
# Add 20 children
|
|
child_ids: list[str] = []
|
|
for i in range(1, 21):
|
|
cid = str(ULID())
|
|
child_ids.append(cid)
|
|
superseded = str(ULID()) if i % 5 == 0 else None
|
|
self.decisions.append(
|
|
_make_decision(
|
|
i, parent_id=root_id, decision_id=cid, superseded_by=superseded
|
|
)
|
|
)
|
|
|
|
def time_tree_build(self) -> None:
|
|
"""Benchmark tree construction."""
|
|
build_decision_tree(self.decisions)
|
|
|
|
def time_tree_render_table(self) -> None:
|
|
"""Benchmark tree table rendering."""
|
|
tree = build_decision_tree(self.decisions)
|
|
format_output(tree, "table")
|
|
|
|
def time_tree_filter_superseded(self) -> None:
|
|
"""Benchmark tree with superseded filtering."""
|
|
build_decision_tree(self.decisions, show_superseded=False)
|