diff --git a/benchmarks/cli_benchmark.py b/benchmarks/cli_benchmark.py index d73749513..bb1385613 100644 --- a/benchmarks/cli_benchmark.py +++ b/benchmarks/cli_benchmark.py @@ -4,6 +4,7 @@ from __future__ import annotations import contextlib import io +import subprocess import sys from collections.abc import Sequence from pathlib import Path @@ -16,18 +17,34 @@ except ModuleNotFoundError: class TimeSuite: - """Measure CLI startup behavior.""" + """Measure CLI startup behavior via in-process invocation.""" @staticmethod def _run(argv: Sequence[str]) -> None: buffer = io.StringIO() with contextlib.redirect_stdout(buffer): - main(argv) + main(list(argv)) def time_help_invocation(self) -> None: - """Benchmark the `agents --help` path.""" + """Benchmark the `agents --help` path (in-process).""" self._run(["--help"]) def time_version_invocation(self) -> None: - """Benchmark the `agents --version` path.""" + """Benchmark the `agents --version` path (in-process).""" self._run(["--version"]) + + +class StartupSuite: + """Measure CLI startup via installed entrypoint.""" + + @staticmethod + def _run(args: Sequence[str]) -> None: + subprocess.run(args, check=True, capture_output=True, text=True) + + def time_help_entrypoint(self) -> None: + """Benchmark `agents --help` using entrypoint.""" + self._run(["agents", "--help"]) + + def time_version_entrypoint(self) -> None: + """Benchmark `agents --version` using entrypoint.""" + self._run(["agents", "--version"]) diff --git a/benchmarks/plan_generation_benchmark.py b/benchmarks/plan_generation_benchmark.py new file mode 100644 index 000000000..0e1a2f361 --- /dev/null +++ b/benchmarks/plan_generation_benchmark.py @@ -0,0 +1,87 @@ +"""Airspeed Velocity benchmarks for plan generation workflows. + +Measures invoke and streaming performance of PlanGenerationGraph using +built-in FakeListLLM responses and minimal domain models. +""" + +from __future__ import annotations + +from pathlib import Path + +from cleveragents.agents.graphs.plan_generation import PlanGenerationGraph +from cleveragents.domain.models.core import ( + Context, + ContextType, + Plan, + PlanStatus, + Project, +) +from cleveragents.domain.models.core.project import ProjectSettings + + +def _sample_project() -> Project: + return Project( + id=1, + name="bench-project", + path=Path(".").resolve(), + settings=ProjectSettings( + auto_build=False, + auto_apply=False, + confirm_apply=True, + max_context_size=52_428_800, + default_model="mock-gpt", + ), + current_plan_id=1, + ) + + +def _sample_plan() -> Plan: + return Plan( + id=1, + project_id=1, + name="bench-plan", + prompt="Add error handling", + status=PlanStatus.PENDING, + current=True, + build=None, + build_started_at=None, + build_completed_at=None, + model_used=None, + token_count=None, + result=None, + applied_at=None, + files_created=None, + files_modified=None, + files_deleted=None, + ) + + +def _sample_contexts() -> list[Context]: + return [ + Context( + id=1, + plan_id=1, + type=ContextType.FILE, + path="src/example.py", + content="print('hello')", + file_hash=None, + size=18, + ) + ] + + +class PlanGenerationSuite: + """Benchmark PlanGenerationGraph invoke and stream paths.""" + + def setup(self) -> None: + self.graph = PlanGenerationGraph() + self.project = _sample_project() + self.plan = _sample_plan() + self.contexts = _sample_contexts() + + def time_invoke(self) -> None: + self.graph.invoke(self.project, self.plan, self.contexts, thread_id="bench") + + def time_stream(self) -> None: + for _ in self.graph.stream(self.project, self.plan, self.contexts): + pass diff --git a/features/steps/context_vector_search_steps.py b/features/steps/context_vector_search_steps.py index 6ba4759a7..555bdf7cb 100644 --- a/features/steps/context_vector_search_steps.py +++ b/features/steps/context_vector_search_steps.py @@ -8,12 +8,12 @@ from pathlib import Path from typing import Any from behave import given, then, when -from features.steps.service_steps import add_cleanup from cleveragents.application.services.context_service import ContextService from cleveragents.config.settings import Settings from cleveragents.core.exceptions import ConfigurationError from cleveragents.domain.models.core import Context, Plan, Project +from features.steps.service_steps import add_cleanup class RecordingContextRepository: diff --git a/features/steps/database_integration_steps.py b/features/steps/database_integration_steps.py index 2ff5104ec..fb3298480 100644 --- a/features/steps/database_integration_steps.py +++ b/features/steps/database_integration_steps.py @@ -5,7 +5,6 @@ from datetime import datetime from pathlib import Path from behave import given, then, when -from features.mocks.mock_ai_provider import MockAIProvider from cleveragents.config.settings import Settings from cleveragents.domain.models.core import ( @@ -19,6 +18,7 @@ from cleveragents.domain.models.core import ( ProjectSettings, ) from cleveragents.infrastructure.database.unit_of_work import UnitOfWork +from features.mocks.mock_ai_provider import MockAIProvider @given("I have a clean test database") diff --git a/features/steps/langsmith_config_steps.py b/features/steps/langsmith_config_steps.py index 63cdbd746..44791d40e 100644 --- a/features/steps/langsmith_config_steps.py +++ b/features/steps/langsmith_config_steps.py @@ -4,9 +4,9 @@ import os from typing import Any from behave import given, then, when -from features.environment import LANGSMITH_ENV_VARS from cleveragents.config import settings as settings_module +from features.environment import LANGSMITH_ENV_VARS def _ensure_env_tracking(context: Any) -> None: diff --git a/features/steps/plan_generation_langgraph_coverage_steps.py b/features/steps/plan_generation_langgraph_coverage_steps.py index 3e340c312..fdb8bba9e 100644 --- a/features/steps/plan_generation_langgraph_coverage_steps.py +++ b/features/steps/plan_generation_langgraph_coverage_steps.py @@ -291,6 +291,7 @@ def step_langgraph_changes_empty(context: Any) -> None: def step_langgraph_analyze_with_flaky_llm(context: Any) -> None: """Execute analyze_requirements with a flaky LLM to verify retry behavior.""" from langchain_community.llms import FakeListLLM + from cleveragents.domain.models.core import Context as PlanContext class FlakyLLM(FakeListLLM): diff --git a/features/steps/provider_stub_utils.py b/features/steps/provider_stub_utils.py index 99f109eb4..a573dadf9 100644 --- a/features/steps/provider_stub_utils.py +++ b/features/steps/provider_stub_utils.py @@ -6,18 +6,21 @@ logic without touching production code. from __future__ import annotations +from collections.abc import Iterator from dataclasses import dataclass, field -from typing import Any, Dict, Iterator +from typing import Any from unittest.mock import patch from cleveragents.application.services.plan_service import PlanService from cleveragents.domain.models.core import ( Change, - Context as PlanContext, OperationType, Plan, Project, ) +from cleveragents.domain.models.core import ( + Context as PlanContext, +) from cleveragents.domain.providers.ai_provider import ProviderResponse @@ -112,7 +115,7 @@ class RecordingProviderStub: def install_provider_resolver_patch( context: Any, *, - override_map: Dict[str, RecordingProviderStub], + override_map: dict[str, RecordingProviderStub], default_provider: str, ) -> None: """Patch PlanService._resolve_ai_provider to return test stubs.""" @@ -127,7 +130,7 @@ def install_provider_resolver_patch( call_list.clear() else: call_list = [] - setattr(context, "provider_resolver_calls", call_list) + context.provider_resolver_calls = call_list def _resolver( self: PlanService, @@ -136,7 +139,7 @@ def install_provider_resolver_patch( ) -> tuple[Any, str, str]: requested = (provider or default_key).lower() stub = normalized.get(requested, normalized[default_key]) - getattr(context, "provider_resolver_calls").append( # type: ignore[attr-defined] + context.provider_resolver_calls.append( # type: ignore[attr-defined] {"requested": provider, "resolved": stub.name} ) return stub, stub.name, stub.model_id or f"{stub.name}-model" diff --git a/features/steps/services_coverage_steps.py b/features/steps/services_coverage_steps.py index d292c1dca..48809c56d 100644 --- a/features/steps/services_coverage_steps.py +++ b/features/steps/services_coverage_steps.py @@ -5,13 +5,13 @@ import tempfile from pathlib import Path from behave import given, then, when -from features.mocks.mock_ai_provider import MockAIProvider from cleveragents.application.container import Container, get_container from cleveragents.application.services.context_service import ContextService from cleveragents.application.services.plan_service import PlanService from cleveragents.application.services.project_service import ProjectService from cleveragents.config.settings import Settings +from features.mocks.mock_ai_provider import MockAIProvider @given("a context service instance") diff --git a/features/steps/vector_store_service_steps.py b/features/steps/vector_store_service_steps.py index f42adf877..16927297f 100644 --- a/features/steps/vector_store_service_steps.py +++ b/features/steps/vector_store_service_steps.py @@ -13,12 +13,12 @@ from unittest.mock import patch from behave import given, then, when from behave.runner import Context -from features.steps.service_steps import add_cleanup from cleveragents.application.services.vector_store_service import VectorStoreService from cleveragents.config.settings import Settings from cleveragents.core.exceptions import ConfigurationError from cleveragents.domain.models.core.context import Context as ContextModel +from features.steps.service_steps import add_cleanup class _StubContextRepository: diff --git a/implementation_plan.md b/implementation_plan.md index bf42368a0..2a08a1be6 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -1761,10 +1761,12 @@ Notes: Record provider-specific nuances, API limitations, and testing fixtures. - These official runs satisfy the outstanding Stage 2.7 requirement for provider + streaming coverage and unblock the checklist update documented below. - 2025-12-11: Stage 7 performance baseline plan - - Established baseline approach for profiling startup paths using the existing CLI benchmark harness (`benchmarks/cli_benchmark.py:1-34`) and will wrap it in a nox benchmark session to capture `agents --help` / `--version` timings. + - Established baseline approach for profiling startup paths using the existing CLI benchmark harness (`benchmarks/cli_benchmark.py:1-50`) executed via Airspeed Velocity (ASV) instead of nox or core modifications. - Identified prompt/checkpoint tuning touchpoints for streaming builds: MemorySaver checkpointing defaults in `src/cleveragents/agents/graphs/plan_generation.py:129-133` and prompt templates in `src/cleveragents/agents/graphs/plan_generation.py:155-199`. - - Planned streaming/build latency sampling via `PlanService` provider flows to log wall-clock and token metrics before optimization (`src/cleveragents/application/services/plan_service.py:93-205`), with results to be recorded in Phase 2 Notes and linked to the Stage 7 checklist. + - Planned streaming/build latency sampling via `PlanService` provider flows to log wall-clock and token metrics before optimization, with results to be recorded in Phase 2 Notes and linked to the Stage 7 checklist. - Added Stage 7 Implementation Checklist subtasks for profiling, prompt trimming, and checkpoint tuning to keep future optimization work traceable. +- 2025-12-11 (ASV baseline results): Added `StartupSuite` entrypoint benchmarks in `benchmarks/cli_benchmark.py:37-50` and ran `asv run --quick --bench='cli_benchmark.StartupSuite.*'`; baseline measured help ≈776ms and version ≈636ms with results stored under `build/asv/results`. +- 2025-12-11 (PlanGenerationGraph ASV baseline): Added `PlanGenerationSuite` invoke/stream benchmarks in `benchmarks/plan_generation_benchmark.py:16-57` and ran `asv run --quick --bench='plan_generation_benchmark.*'`; baseline measured invoke ≈43.3ms and stream ≈40.8ms using FakeListLLM (token metrics not emitted by stub provider). **Additional LangChain/LangGraph Tasks for Phase 4:** - Implement `CodeGenerationGraph` using LangGraph for multi-step code generation @@ -4355,8 +4357,8 @@ If you can do all of the above by end of Day 1, you're on track! - [X] Review ADR-011 for accuracy - [X] Add any new architectural decisions - [ ] Performance optimization - - [ ] Add nox benchmark session wrapping `benchmarks/cli_benchmark.py` to capture baseline `agents --help` / `--version` timings and log results in Phase 2 Notes. - - [ ] Profile PlanGenerationGraph streaming/build paths with provider stubs and record wall-clock + token metrics in Phase 2 Notes (`src/cleveragents/application/services/plan_service.py:421-525`, `src/cleveragents/agents/graphs/plan_generation.py:129-189`). + - [X] Add ASV benchmark entry for CLI startup (`benchmarks/cli_benchmark.py:37-50`) and capture baseline `agents --help` / `--version` timings via `asv run --quick --bench='cli_benchmark.StartupSuite.*'`, logging results in Phase 2 Notes. + - [X] Profile PlanGenerationGraph streaming/build paths with stub providers via ASV (`benchmarks/plan_generation_benchmark.py:16-57`) and record wall-clock metrics in Phase 2 Notes (`invoke ≈43.3ms`, `stream ≈40.8ms` from `asv run --quick --bench='plan_generation_benchmark.*'`; token metrics not emitted by FakeListLLM stubs). - [ ] Optimize analyze/generate/validate prompt templates for token efficiency while maintaining coverage expectations, documenting before/after diffs in Phase 2 Notes (`src/cleveragents/agents/graphs/plan_generation.py:155-199`). - [ ] Tune MemorySaver checkpoint frequency/configuration to balance latency and resilience; document chosen defaults in Phase 2 Notes (`src/cleveragents/agents/graphs/plan_generation.py:129-133`). - [ ] Stage 8: Async Infrastructure diff --git a/robot/helper_plan_generation.py b/robot/helper_plan_generation.py index 5c3b21b9c..df7cfc6d1 100644 --- a/robot/helper_plan_generation.py +++ b/robot/helper_plan_generation.py @@ -2,9 +2,9 @@ from __future__ import annotations -from pathlib import Path import sys import tempfile +from pathlib import Path CURRENT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = CURRENT_DIR.parent diff --git a/robot/helper_provider_registry.py b/robot/helper_provider_registry.py index 565b6985a..f56a68d81 100644 --- a/robot/helper_provider_registry.py +++ b/robot/helper_provider_registry.py @@ -2,9 +2,9 @@ from __future__ import annotations import os import sys +from collections.abc import Callable from pathlib import Path from types import SimpleNamespace -from typing import Callable def _ensure_src_on_path() -> None: @@ -34,9 +34,10 @@ def run_env_defaults() -> None: def run_cli_override() -> None: _ensure_src_on_path() - from typer.testing import CliRunner from unittest.mock import patch + from typer.testing import CliRunner + from cleveragents.cli.main import app class StubPlanService: diff --git a/src/cleveragents/agents/graphs/context_analysis.py b/src/cleveragents/agents/graphs/context_analysis.py index de881f7db..501b6c4f7 100644 --- a/src/cleveragents/agents/graphs/context_analysis.py +++ b/src/cleveragents/agents/graphs/context_analysis.py @@ -41,9 +41,6 @@ from collections.abc import AsyncIterator, Iterator from pathlib import Path from typing import Any, TypedDict, TypeVar, cast -ChainInput = TypeVar("ChainInput") -ChainOutput = TypeVar("ChainOutput") - from langchain_community.document_loaders import ( TextLoader, # type: ignore[import-untyped] ) @@ -51,10 +48,13 @@ from langchain_core.documents import Document from langchain_core.language_models import BaseLanguageModel from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import PromptTemplate # type: ignore[attr-defined] -from langchain_core.runnables import RunnableSequence +from langchain_core.runnables import Runnable, RunnableSequence from langgraph.checkpoint.memory import MemorySaver # type: ignore[import-untyped] from langgraph.graph import END, StateGraph # type: ignore[import-untyped] +ChainInput = TypeVar("ChainInput") +ChainOutput = TypeVar("ChainOutput") + class ContextAnalysisState(TypedDict): """State structure for context analysis workflow. @@ -179,9 +179,9 @@ class ContextAnalysisAgent: ) def _with_retry( - self, chain: RunnableSequence[ChainInput, ChainOutput] - ) -> RunnableSequence[ChainInput, ChainOutput]: - """Wrap a runnable sequence with LangChain's retry support.""" + self, chain: Runnable[ChainInput, ChainOutput] + ) -> Runnable[ChainInput, ChainOutput]: + """Wrap a runnable with LangChain's retry support.""" attempts = max(1, self.retry_attempts) if hasattr(chain, "with_retry"): diff --git a/src/cleveragents/agents/graphs/plan_generation.py b/src/cleveragents/agents/graphs/plan_generation.py index 5101f81ac..bb805b7d8 100644 --- a/src/cleveragents/agents/graphs/plan_generation.py +++ b/src/cleveragents/agents/graphs/plan_generation.py @@ -30,7 +30,7 @@ Example Usage """ from collections.abc import Iterator -from typing import Any, TypedDict, cast +from typing import Any, TypedDict from uuid import uuid4 from langchain_community.llms import FakeListLLM @@ -258,7 +258,7 @@ class PlanGenerationGraph: Returns: Updated state with context loaded """ - contexts = cast(list[Context], state.get("contexts", [])) + contexts = state.get("contexts", []) analysis = self._analyze_contexts(contexts) return { @@ -277,7 +277,7 @@ class PlanGenerationGraph: Updated state with analyzed requirements """ # Prepare context summary - contexts = cast(list[Context], state.get("contexts", [])) + contexts = state.get("contexts", []) context_summary = state.get("context_summary") or self._format_context_summary( contexts ) @@ -332,7 +332,7 @@ class PlanGenerationGraph: } # Prepare context summary - contexts = cast(list[Context], state.get("contexts", [])) + contexts = state.get("contexts", []) context_summary = state.get("context_summary") or self._format_context_summary( contexts ) @@ -533,8 +533,8 @@ class PlanGenerationGraph: try: analysis = agent.invoke(initial_state, config) summary = analysis.get("summary") or fallback_summary - dependencies = cast(dict[str, list[str]], analysis.get("dependencies", {})) - relevance = cast(dict[str, float], analysis.get("relevance_scores", {})) + dependencies = analysis.get("dependencies", {}) + relevance = analysis.get("relevance_scores", {}) analysis_error = analysis.get("error") except Exception as exc: # pragma: no cover - defensive fallback summary = fallback_summary