freemo
a074b4846f
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 19s
CI / quality (pull_request) Successful in 28s
CI / security (pull_request) Successful in 50s
CI / typecheck (pull_request) Successful in 56s
CI / integration_tests (pull_request) Successful in 4m51s
CI / unit_tests (pull_request) Successful in 19m29s
CI / docker (pull_request) Successful in 39s
CI / benchmark-regression (pull_request) Successful in 26m10s
CI / coverage (pull_request) Successful in 47m42s
CI / lint (push) Successful in 13s
CI / quality (push) Successful in 17s
CI / build (push) Successful in 23s
CI / typecheck (push) Successful in 30s
CI / security (push) Successful in 30s
CI / benchmark-regression (push) Has been skipped
CI / integration_tests (push) Successful in 2m52s
CI / unit_tests (push) Successful in 10m8s
CI / docker (push) Successful in 1m19s
CI / benchmark-publish (push) Successful in 11m54s
CI / coverage (push) Failing after 39m51s
Remove FakeListLLM as a silent fallback in agent graph constructors (plan_generation.py, context_analysis.py, auto_debug.py). All three now raise ValueError when llm=None, making missing-provider errors explicit. Add Settings.mock_providers flag and validate_provider_availability() method. Update container.get_ai_provider() to check Settings.mock_providers first, with env-var fallback for backward compatibility. Add resolve_provider_by_name() helper to the provider registry and export it from cleveragents.providers. Add structlog trace logging to ProviderRegistry.get_default_provider_type() to record selection reasoning. Update all existing behave step files, robot tests, and benchmarks that relied on the implicit FakeListLLM default to pass an explicit LLM instance instead. Add new BDD tests (features/provider_fixes.feature with 17 scenarios), Robot Framework integration tests (robot/provider_detection_smoke.robot), and ASV benchmarks (benchmarks/provider_selection_bench.py). ISSUES CLOSED: #323
109 lines
2.9 KiB
Python
109 lines
2.9 KiB
Python
"""Airspeed Velocity benchmarks for plan generation workflows.
|
|
|
|
Measures invoke and streaming performance of PlanGenerationGraph using
|
|
built-in FakeListLLM responses and minimal domain models.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import tiktoken
|
|
from langchain_community.llms import FakeListLLM
|
|
|
|
from cleveragents.agents.graphs.plan_generation import PlanGenerationGraph
|
|
from cleveragents.domain.models.core import (
|
|
Context,
|
|
ContextType,
|
|
Plan,
|
|
PlanStatus,
|
|
Project,
|
|
)
|
|
from cleveragents.domain.models.core.project_legacy import ProjectSettings
|
|
|
|
|
|
def _sample_project() -> Project:
|
|
return Project(
|
|
id=1,
|
|
name="bench-project",
|
|
path=Path(".").resolve(),
|
|
settings=ProjectSettings(
|
|
auto_build=False,
|
|
auto_apply=False,
|
|
confirm_apply=True,
|
|
max_context_size=52_428_800,
|
|
default_model="mock-gpt",
|
|
),
|
|
current_plan_id=1,
|
|
)
|
|
|
|
|
|
def _sample_plan() -> Plan:
|
|
return Plan(
|
|
id=1,
|
|
project_id=1,
|
|
name="bench-plan",
|
|
prompt="Add error handling",
|
|
status=PlanStatus.PENDING,
|
|
current=True,
|
|
build=None,
|
|
build_started_at=None,
|
|
build_completed_at=None,
|
|
model_used=None,
|
|
token_count=None,
|
|
result=None,
|
|
applied_at=None,
|
|
files_created=None,
|
|
files_modified=None,
|
|
files_deleted=None,
|
|
)
|
|
|
|
|
|
def _sample_contexts() -> list[Context]:
|
|
return [
|
|
Context(
|
|
id=1,
|
|
plan_id=1,
|
|
type=ContextType.FILE,
|
|
path="src/example.py",
|
|
content="print('hello')",
|
|
file_hash=None,
|
|
size=18,
|
|
)
|
|
]
|
|
|
|
|
|
class PlanGenerationSuite:
|
|
"""Benchmark PlanGenerationGraph invoke and stream paths."""
|
|
|
|
def setup(self) -> None:
|
|
self.graph = PlanGenerationGraph(
|
|
llm=FakeListLLM(responses=["bench response"] * 3)
|
|
)
|
|
self.project = _sample_project()
|
|
self.plan = _sample_plan()
|
|
self.contexts = _sample_contexts()
|
|
|
|
enc = tiktoken.get_encoding("cl100k_base")
|
|
self.prompt_tokens = {
|
|
"analyze": len(enc.encode(self.graph.analyze_prompt.template)),
|
|
"generate": len(enc.encode(self.graph.generate_prompt.template)),
|
|
"validate": len(enc.encode(self.graph.validate_prompt.template)),
|
|
}
|
|
|
|
def time_invoke(self) -> None:
|
|
self.graph.invoke(self.project, self.plan, self.contexts, thread_id="bench")
|
|
|
|
def time_stream(self) -> None:
|
|
for _ in self.graph.stream(self.project, self.plan, self.contexts):
|
|
pass
|
|
|
|
def track_analyze_prompt_tokens(self) -> int:
|
|
return self.prompt_tokens["analyze"]
|
|
|
|
def track_generate_prompt_tokens(self) -> int:
|
|
return self.prompt_tokens["generate"]
|
|
|
|
def track_validate_prompt_tokens(self) -> int:
|
|
return self.prompt_tokens["validate"]
|