Tests: Added benchmarks for PlanGenerationGraph

benchmarks/cli_benchmark.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import contextlib
 import io
+import subprocess
 import sys
 from collections.abc import Sequence
 from pathlib import Path
@@ -16,18 +17,34 @@ except ModuleNotFoundError:
 
 
 class TimeSuite:
-    """Measure CLI startup behavior."""
+    """Measure CLI startup behavior via in-process invocation."""
 
     @staticmethod
     def _run(argv: Sequence[str]) -> None:
         buffer = io.StringIO()
         with contextlib.redirect_stdout(buffer):
-            main(argv)
+            main(list(argv))
 
     def time_help_invocation(self) -> None:
-        """Benchmark the `agents --help` path."""
+        """Benchmark the `agents --help` path (in-process)."""
         self._run(["--help"])
 
     def time_version_invocation(self) -> None:
-        """Benchmark the `agents --version` path."""
+        """Benchmark the `agents --version` path (in-process)."""
         self._run(["--version"])
+
+
+class StartupSuite:
+    """Measure CLI startup via installed entrypoint."""
+
+    @staticmethod
+    def _run(args: Sequence[str]) -> None:
+        subprocess.run(args, check=True, capture_output=True, text=True)
+
+    def time_help_entrypoint(self) -> None:
+        """Benchmark `agents --help` using entrypoint."""
+        self._run(["agents", "--help"])
+
+    def time_version_entrypoint(self) -> None:
+        """Benchmark `agents --version` using entrypoint."""
+        self._run(["agents", "--version"])

benchmarks/plan_generation_benchmark.py

@@ -0,0 +1,87 @@
+"""Airspeed Velocity benchmarks for plan generation workflows.
+
+Measures invoke and streaming performance of PlanGenerationGraph using
+built-in FakeListLLM responses and minimal domain models.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from cleveragents.agents.graphs.plan_generation import PlanGenerationGraph
+from cleveragents.domain.models.core import (
+    Context,
+    ContextType,
+    Plan,
+    PlanStatus,
+    Project,
+)
+from cleveragents.domain.models.core.project import ProjectSettings
+
+
+def _sample_project() -> Project:
+    return Project(
+        id=1,
+        name="bench-project",
+        path=Path(".").resolve(),
+        settings=ProjectSettings(
+            auto_build=False,
+            auto_apply=False,
+            confirm_apply=True,
+            max_context_size=52_428_800,
+            default_model="mock-gpt",
+        ),
+        current_plan_id=1,
+    )
+
+
+def _sample_plan() -> Plan:
+    return Plan(
+        id=1,
+        project_id=1,
+        name="bench-plan",
+        prompt="Add error handling",
+        status=PlanStatus.PENDING,
+        current=True,
+        build=None,
+        build_started_at=None,
+        build_completed_at=None,
+        model_used=None,
+        token_count=None,
+        result=None,
+        applied_at=None,
+        files_created=None,
+        files_modified=None,
+        files_deleted=None,
+    )
+
+
+def _sample_contexts() -> list[Context]:
+    return [
+        Context(
+            id=1,
+            plan_id=1,
+            type=ContextType.FILE,
+            path="src/example.py",
+            content="print('hello')",
+            file_hash=None,
+            size=18,
+        )
+    ]
+
+
+class PlanGenerationSuite:
+    """Benchmark PlanGenerationGraph invoke and stream paths."""
+
+    def setup(self) -> None:
+        self.graph = PlanGenerationGraph()
+        self.project = _sample_project()
+        self.plan = _sample_plan()
+        self.contexts = _sample_contexts()
+
+    def time_invoke(self) -> None:
+        self.graph.invoke(self.project, self.plan, self.contexts, thread_id="bench")
+
+    def time_stream(self) -> None:
+        for _ in self.graph.stream(self.project, self.plan, self.contexts):
+            pass

features/steps/context_vector_search_steps.py

@@ -8,12 +8,12 @@ from pathlib import Path
 from typing import Any
 
 from behave import given, then, when
-from features.steps.service_steps import add_cleanup
 
 from cleveragents.application.services.context_service import ContextService
 from cleveragents.config.settings import Settings
 from cleveragents.core.exceptions import ConfigurationError
 from cleveragents.domain.models.core import Context, Plan, Project
+from features.steps.service_steps import add_cleanup
 
 
 class RecordingContextRepository:

features/steps/database_integration_steps.py

@@ -5,7 +5,6 @@ from datetime import datetime
 from pathlib import Path
 
 from behave import given, then, when
-from features.mocks.mock_ai_provider import MockAIProvider
 
 from cleveragents.config.settings import Settings
 from cleveragents.domain.models.core import (
@@ -19,6 +18,7 @@ from cleveragents.domain.models.core import (
     ProjectSettings,
 )
 from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
+from features.mocks.mock_ai_provider import MockAIProvider
 
 
 @given("I have a clean test database")

features/steps/langsmith_config_steps.py

@@ -4,9 +4,9 @@ import os
 from typing import Any
 
 from behave import given, then, when
-from features.environment import LANGSMITH_ENV_VARS
 
 from cleveragents.config import settings as settings_module
+from features.environment import LANGSMITH_ENV_VARS
 
 
 def _ensure_env_tracking(context: Any) -> None:

features/steps/plan_generation_langgraph_coverage_steps.py

@@ -291,6 +291,7 @@ def step_langgraph_changes_empty(context: Any) -> None:
 def step_langgraph_analyze_with_flaky_llm(context: Any) -> None:
     """Execute analyze_requirements with a flaky LLM to verify retry behavior."""
     from langchain_community.llms import FakeListLLM
+
     from cleveragents.domain.models.core import Context as PlanContext
 
     class FlakyLLM(FakeListLLM):

features/steps/provider_stub_utils.py

@@ -6,18 +6,21 @@ logic without touching production code.
 
 from __future__ import annotations
 
+from collections.abc import Iterator
 from dataclasses import dataclass, field
-from typing import Any, Dict, Iterator
+from typing import Any
 from unittest.mock import patch
 
 from cleveragents.application.services.plan_service import PlanService
 from cleveragents.domain.models.core import (
     Change,
-    Context as PlanContext,
     OperationType,
     Plan,
     Project,
 )
+from cleveragents.domain.models.core import (
+    Context as PlanContext,
+)
 from cleveragents.domain.providers.ai_provider import ProviderResponse
 
 
@@ -112,7 +115,7 @@ class RecordingProviderStub:
 def install_provider_resolver_patch(
     context: Any,
     *,
-    override_map: Dict[str, RecordingProviderStub],
+    override_map: dict[str, RecordingProviderStub],
     default_provider: str,
 ) -> None:
     """Patch PlanService._resolve_ai_provider to return test stubs."""
@@ -127,7 +130,7 @@ def install_provider_resolver_patch(
         call_list.clear()
     else:
         call_list = []
-        setattr(context, "provider_resolver_calls", call_list)
+        context.provider_resolver_calls = call_list
 
     def _resolver(
         self: PlanService,
@@ -136,7 +139,7 @@ def install_provider_resolver_patch(
     ) -> tuple[Any, str, str]:
         requested = (provider or default_key).lower()
         stub = normalized.get(requested, normalized[default_key])
-        getattr(context, "provider_resolver_calls").append(  # type: ignore[attr-defined]
+        context.provider_resolver_calls.append(  # type: ignore[attr-defined]
             {"requested": provider, "resolved": stub.name}
         )
         return stub, stub.name, stub.model_id or f"{stub.name}-model"

features/steps/services_coverage_steps.py

@@ -5,13 +5,13 @@ import tempfile
 from pathlib import Path
 
 from behave import given, then, when
-from features.mocks.mock_ai_provider import MockAIProvider
 
 from cleveragents.application.container import Container, get_container
 from cleveragents.application.services.context_service import ContextService
 from cleveragents.application.services.plan_service import PlanService
 from cleveragents.application.services.project_service import ProjectService
 from cleveragents.config.settings import Settings
+from features.mocks.mock_ai_provider import MockAIProvider
 
 
 @given("a context service instance")

features/steps/vector_store_service_steps.py

@@ -13,12 +13,12 @@ from unittest.mock import patch
 
 from behave import given, then, when
 from behave.runner import Context
-from features.steps.service_steps import add_cleanup
 
 from cleveragents.application.services.vector_store_service import VectorStoreService
 from cleveragents.config.settings import Settings
 from cleveragents.core.exceptions import ConfigurationError
 from cleveragents.domain.models.core.context import Context as ContextModel
+from features.steps.service_steps import add_cleanup
 
 
 class _StubContextRepository:

implementation_plan.md

@@ -1761,10 +1761,12 @@ Notes: Record provider-specific nuances, API limitations, and testing fixtures.
 - These official runs satisfy the outstanding Stage 2.7 requirement for provider + streaming coverage and unblock the checklist update documented below.
 
 - 2025-12-11: Stage 7 performance baseline plan
-  - Established baseline approach for profiling startup paths using the existing CLI benchmark harness (`benchmarks/cli_benchmark.py:1-34`) and will wrap it in a nox benchmark session to capture `agents --help` / `--version` timings.
+  - Established baseline approach for profiling startup paths using the existing CLI benchmark harness (`benchmarks/cli_benchmark.py:1-50`) executed via Airspeed Velocity (ASV) instead of nox or core modifications.
   - Identified prompt/checkpoint tuning touchpoints for streaming builds: MemorySaver checkpointing defaults in `src/cleveragents/agents/graphs/plan_generation.py:129-133` and prompt templates in `src/cleveragents/agents/graphs/plan_generation.py:155-199`.
-  - Planned streaming/build latency sampling via `PlanService` provider flows to log wall-clock and token metrics before optimization (`src/cleveragents/application/services/plan_service.py:93-205`), with results to be recorded in Phase 2 Notes and linked to the Stage 7 checklist.
+  - Planned streaming/build latency sampling via `PlanService` provider flows to log wall-clock and token metrics before optimization, with results to be recorded in Phase 2 Notes and linked to the Stage 7 checklist.
   - Added Stage 7 Implementation Checklist subtasks for profiling, prompt trimming, and checkpoint tuning to keep future optimization work traceable.
+- 2025-12-11 (ASV baseline results): Added `StartupSuite` entrypoint benchmarks in `benchmarks/cli_benchmark.py:37-50` and ran `asv run --quick --bench='cli_benchmark.StartupSuite.*'`; baseline measured help ≈776ms and version ≈636ms with results stored under `build/asv/results`.
+- 2025-12-11 (PlanGenerationGraph ASV baseline): Added `PlanGenerationSuite` invoke/stream benchmarks in `benchmarks/plan_generation_benchmark.py:16-57` and ran `asv run --quick --bench='plan_generation_benchmark.*'`; baseline measured invoke ≈43.3ms and stream ≈40.8ms using FakeListLLM (token metrics not emitted by stub provider).
 
 **Additional LangChain/LangGraph Tasks for Phase 4:**
 - Implement `CodeGenerationGraph` using LangGraph for multi-step code generation
@@ -4355,8 +4357,8 @@ If you can do all of the above by end of Day 1, you're on track!
     - [X] Review ADR-011 for accuracy
     - [X] Add any new architectural decisions
     - [ ] Performance optimization
-      - [ ] Add nox benchmark session wrapping `benchmarks/cli_benchmark.py` to capture baseline `agents --help` / `--version` timings and log results in Phase 2 Notes.
-      - [ ] Profile PlanGenerationGraph streaming/build paths with provider stubs and record wall-clock + token metrics in Phase 2 Notes (`src/cleveragents/application/services/plan_service.py:421-525`, `src/cleveragents/agents/graphs/plan_generation.py:129-189`).
+      - [X] Add ASV benchmark entry for CLI startup (`benchmarks/cli_benchmark.py:37-50`) and capture baseline `agents --help` / `--version` timings via `asv run --quick --bench='cli_benchmark.StartupSuite.*'`, logging results in Phase 2 Notes.
+      - [X] Profile PlanGenerationGraph streaming/build paths with stub providers via ASV (`benchmarks/plan_generation_benchmark.py:16-57`) and record wall-clock metrics in Phase 2 Notes (`invoke ≈43.3ms`, `stream ≈40.8ms` from `asv run --quick --bench='plan_generation_benchmark.*'`; token metrics not emitted by FakeListLLM stubs).
       - [ ] Optimize analyze/generate/validate prompt templates for token efficiency while maintaining coverage expectations, documenting before/after diffs in Phase 2 Notes (`src/cleveragents/agents/graphs/plan_generation.py:155-199`).
       - [ ] Tune MemorySaver checkpoint frequency/configuration to balance latency and resilience; document chosen defaults in Phase 2 Notes (`src/cleveragents/agents/graphs/plan_generation.py:129-133`).
   - [ ] Stage 8: Async Infrastructure

robot/helper_plan_generation.py

@@ -2,9 +2,9 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 import sys
 import tempfile
+from pathlib import Path
 
 CURRENT_DIR = Path(__file__).resolve().parent
 PROJECT_ROOT = CURRENT_DIR.parent

robot/helper_provider_registry.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import os
 import sys
+from collections.abc import Callable
 from pathlib import Path
 from types import SimpleNamespace
-from typing import Callable
 
 
 def _ensure_src_on_path() -> None:
@@ -34,9 +34,10 @@ def run_env_defaults() -> None:
 
 def run_cli_override() -> None:
     _ensure_src_on_path()
-    from typer.testing import CliRunner
     from unittest.mock import patch
 
+    from typer.testing import CliRunner
+
     from cleveragents.cli.main import app
 
     class StubPlanService:

src/cleveragents/agents/graphs/context_analysis.py

@@ -41,9 +41,6 @@ from collections.abc import AsyncIterator, Iterator
 from pathlib import Path
 from typing import Any, TypedDict, TypeVar, cast
 
-ChainInput = TypeVar("ChainInput")
-ChainOutput = TypeVar("ChainOutput")
-
 from langchain_community.document_loaders import (
     TextLoader,  # type: ignore[import-untyped]
 )
@@ -51,10 +48,13 @@ from langchain_core.documents import Document
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import PromptTemplate  # type: ignore[attr-defined]
-from langchain_core.runnables import RunnableSequence
+from langchain_core.runnables import Runnable, RunnableSequence
 from langgraph.checkpoint.memory import MemorySaver  # type: ignore[import-untyped]
 from langgraph.graph import END, StateGraph  # type: ignore[import-untyped]
 
+ChainInput = TypeVar("ChainInput")
+ChainOutput = TypeVar("ChainOutput")
+
 
 class ContextAnalysisState(TypedDict):
     """State structure for context analysis workflow.
@@ -179,9 +179,9 @@ class ContextAnalysisAgent:
         )
 
     def _with_retry(
-        self, chain: RunnableSequence[ChainInput, ChainOutput]
-    ) -> RunnableSequence[ChainInput, ChainOutput]:
-        """Wrap a runnable sequence with LangChain's retry support."""
+        self, chain: Runnable[ChainInput, ChainOutput]
+    ) -> Runnable[ChainInput, ChainOutput]:
+        """Wrap a runnable with LangChain's retry support."""
 
         attempts = max(1, self.retry_attempts)
         if hasattr(chain, "with_retry"):

src/cleveragents/agents/graphs/plan_generation.py

@@ -30,7 +30,7 @@ Example Usage
 """
 
 from collections.abc import Iterator
-from typing import Any, TypedDict, cast
+from typing import Any, TypedDict
 from uuid import uuid4
 
 from langchain_community.llms import FakeListLLM
@@ -258,7 +258,7 @@ class PlanGenerationGraph:
         Returns:
             Updated state with context loaded
         """
-        contexts = cast(list[Context], state.get("contexts", []))
+        contexts = state.get("contexts", [])
         analysis = self._analyze_contexts(contexts)
 
         return {
@@ -277,7 +277,7 @@ class PlanGenerationGraph:
             Updated state with analyzed requirements
         """
         # Prepare context summary
-        contexts = cast(list[Context], state.get("contexts", []))
+        contexts = state.get("contexts", [])
         context_summary = state.get("context_summary") or self._format_context_summary(
             contexts
         )
@@ -332,7 +332,7 @@ class PlanGenerationGraph:
             }
 
         # Prepare context summary
-        contexts = cast(list[Context], state.get("contexts", []))
+        contexts = state.get("contexts", [])
         context_summary = state.get("context_summary") or self._format_context_summary(
             contexts
         )
@@ -533,8 +533,8 @@ class PlanGenerationGraph:
         try:
             analysis = agent.invoke(initial_state, config)
             summary = analysis.get("summary") or fallback_summary
-            dependencies = cast(dict[str, list[str]], analysis.get("dependencies", {}))
-            relevance = cast(dict[str, float], analysis.get("relevance_scores", {}))
+            dependencies = analysis.get("dependencies", {})
+            relevance = analysis.get("relevance_scores", {})
             analysis_error = analysis.get("error")
         except Exception as exc:  # pragma: no cover - defensive fallback
             summary = fallback_summary