cleveragents-core/benchmarks/plan_explain_bench.py

"""ASV benchmarks for plan explain and plan tree operations.

Measures the performance of:
- Explain dict building with various flag combinations
- Decision tree construction and filtering
- Tree table rendering
"""

from __future__ import annotations

import sys
from pathlib import Path

try:
    from cleveragents.cli.commands.plan import (
        _build_explain_dict,
        build_decision_tree,
    )
    from cleveragents.cli.formatting import format_output
    from cleveragents.domain.models.core.decision import (
        ContextSnapshot,
        Decision,
        DecisionType,
        ResourceRef,
    )
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.cli.commands.plan import (
        _build_explain_dict,
        build_decision_tree,
    )
    from cleveragents.cli.formatting import format_output
    from cleveragents.domain.models.core.decision import (
        ContextSnapshot,
        Decision,
        DecisionType,
        ResourceRef,
    )

from ulid import ULID

_PLAN_ID = str(ULID())


def _make_decision(
    seq: int,
    parent_id: str | None = None,
    decision_id: str | None = None,
    dt: DecisionType = DecisionType.STRATEGY_CHOICE,
    superseded_by: str | None = None,
) -> Decision:
    """Build a Decision for benchmarking."""
    if dt == DecisionType.PROMPT_DEFINITION:
        parent_id = None
    elif parent_id is None:
        parent_id = str(ULID())
    return Decision(
        decision_id=decision_id or str(ULID()),
        plan_id=_PLAN_ID,
        parent_decision_id=parent_id,
        sequence_number=seq,
        decision_type=dt,
        question=f"Question {seq}?",
        chosen_option=f"Option {seq}",
        confidence_score=0.8,
        superseded_by=superseded_by,
        context_snapshot=ContextSnapshot(
            hot_context_hash=f"sha256:bench{seq}",
            relevant_resources=[
                ResourceRef(resource_id=str(ULID()), path=f"src/f{seq}.py"),
            ],
        ),
        rationale=f"Rationale for decision {seq}",
        actor_reasoning=f"LLM reasoning for {seq}",
        alternatives_considered=[f"alt-{seq}-a", f"alt-{seq}-b"],
    )


class PlanExplainSuite:
    """Benchmark plan explain formatting."""

    def setup(self) -> None:
        self.decision = _make_decision(0, dt=DecisionType.PROMPT_DEFINITION)

    def time_explain_formatting(self) -> None:
        """Benchmark basic explain dict building."""
        _build_explain_dict(self.decision)

    def time_explain_with_context(self) -> None:
        """Benchmark explain dict building with all flags."""
        _build_explain_dict(
            self.decision,
            show_context=True,
            show_reasoning=True,
            show_alternatives=True,
        )


class PlanTreeSuite:
    """Benchmark plan tree operations."""

    def setup(self) -> None:
        root_id = str(ULID())
        self.root_id = root_id
        self.decisions: list[Decision] = [
            _make_decision(0, dt=DecisionType.PROMPT_DEFINITION, decision_id=root_id),
        ]
        # Add 20 children
        child_ids: list[str] = []
        for i in range(1, 21):
            cid = str(ULID())
            child_ids.append(cid)
            superseded = str(ULID()) if i % 5 == 0 else None
            self.decisions.append(
                _make_decision(
                    i, parent_id=root_id, decision_id=cid, superseded_by=superseded
                )
            )

    def time_tree_build(self) -> None:
        """Benchmark tree construction."""
        build_decision_tree(self.decisions)

    def time_tree_render_table(self) -> None:
        """Benchmark tree table rendering."""
        tree = build_decision_tree(self.decisions)
        format_output(tree, "table")

    def time_tree_filter_superseded(self) -> None:
        """Benchmark tree with superseded filtering."""
        build_decision_tree(self.decisions, show_superseded=False)