Files
placeholder/features/steps/context_analysis_graph_coverage_steps.py
freemo a074b4846f fix(provider): remove FakeListLLM defaults
Remove FakeListLLM as a silent fallback in agent graph constructors
(plan_generation.py, context_analysis.py, auto_debug.py). All three now
raise ValueError when llm=None, making missing-provider errors explicit.

Add Settings.mock_providers flag and validate_provider_availability()
method. Update container.get_ai_provider() to check Settings.mock_providers
first, with env-var fallback for backward compatibility.

Add resolve_provider_by_name() helper to the provider registry and export
it from cleveragents.providers. Add structlog trace logging to
ProviderRegistry.get_default_provider_type() to record selection reasoning.

Update all existing behave step files, robot tests, and benchmarks that
relied on the implicit FakeListLLM default to pass an explicit LLM
instance instead.

Add new BDD tests (features/provider_fixes.feature with 17 scenarios),
Robot Framework integration tests (robot/provider_detection_smoke.robot),
and ASV benchmarks (benchmarks/provider_selection_bench.py).

ISSUES CLOSED: #323
2026-02-27 09:47:10 -05:00

524 lines
19 KiB
Python

"""Step definitions for context analysis graph coverage scenarios.
These steps specifically target uncovered lines in
src/cleveragents/agents/graphs/context_analysis.py as identified
by the build/coverage.xml report.
"""
from __future__ import annotations
import asyncio
import shutil
import tempfile
from pathlib import Path
from typing import Any
from behave import given, then, when
from langchain_community.llms import FakeListLLM
from langchain_core.documents import Document
from cleveragents.agents.graphs.context_analysis import (
ContextAnalysisAgent,
ContextAnalysisState,
)
def _default_context_test_llm() -> FakeListLLM:
"""Create a FakeListLLM for context analysis test purposes."""
return FakeListLLM(
responses=[
"Dependencies: ['os', 'sys', 'pathlib']",
"Relevance: High - contains core functionality",
"Summary: Python module implementing core business logic",
]
)
# Enough responses so every LLM call across the full workflow is satisfied.
_DEFAULT_RESPONSES = [
"Dependencies: ['os', 'sys', 'pathlib']",
"Relevance: 0.8 - core module",
"Summary: Generated context overview",
] * 30
def _make_state(**overrides: Any) -> ContextAnalysisState:
"""Build a minimal valid ContextAnalysisState, with optional overrides."""
state: ContextAnalysisState = {
"file_paths": [],
"documents": [],
"dependencies": {},
"summary": "",
"relevance_scores": {},
"chunks": [],
"error": None,
}
state.update(overrides) # type: ignore[typeddict-item]
return state
def _ensure_temp_dir(context: Any) -> Path:
if not hasattr(context, "graph_temp_dir") or context.graph_temp_dir is None:
context.graph_temp_dir = Path(tempfile.mkdtemp(prefix="ctx-graph-cov-"))
return context.graph_temp_dir
# ---------------------------------------------------------------------------
# Background steps
# ---------------------------------------------------------------------------
@given("the context analysis graph module is importable")
def step_module_importable(context: Any) -> None:
ContextAnalysisAgent # noqa: B018
ContextAnalysisState # noqa: B018
@given("I have a fake LLM configured for context analysis graph tests")
def step_configure_fake_llm(context: Any) -> None:
context.graph_fake_llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
# ---------------------------------------------------------------------------
# Scenario: Agent initialises with an explicitly provided LLM
# Targets lines 115, 117, 125 (the else branch of if llm is None)
# ---------------------------------------------------------------------------
@when("I create a ContextAnalysisAgent with a custom LLM")
def step_create_agent_custom_llm(context: Any) -> None:
custom_llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
context.custom_llm_ref = custom_llm
context.graph_agent = ContextAnalysisAgent(llm=custom_llm)
@then("the agent should use the provided LLM instance")
def step_agent_uses_provided_llm(context: Any) -> None:
assert context.graph_agent.llm is context.custom_llm_ref
@then("the agent should be initialised successfully")
def step_agent_initialised(context: Any) -> None:
assert isinstance(context.graph_agent, ContextAnalysisAgent)
# ---------------------------------------------------------------------------
# Helper: ensure an agent on the context
# ---------------------------------------------------------------------------
def _ensure_agent(context: Any, **kwargs: Any) -> ContextAnalysisAgent:
if hasattr(context, "graph_agent") and context.graph_agent is not None:
return context.graph_agent
llm = kwargs.pop("llm", getattr(context, "graph_fake_llm", None))
if llm is None:
llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
context.graph_agent = ContextAnalysisAgent(llm=llm, **kwargs)
return context.graph_agent
@given("I have a ContextAnalysisAgent for graph coverage")
def step_have_agent(context: Any) -> None:
context.graph_agent = None # force fresh agent
_ensure_agent(context)
@given(
"I have a ContextAnalysisAgent for graph coverage "
"with chunk_size {chunk_size:d} and chunk_overlap {overlap:d}"
)
def step_have_agent_custom_chunks(context: Any, chunk_size: int, overlap: int) -> None:
context.graph_agent = None
_ensure_agent(context, chunk_size=chunk_size, chunk_overlap=overlap)
# ---------------------------------------------------------------------------
# Scenario: Load files discovers missing file paths
# Targets lines 228-236, 248-252
# ---------------------------------------------------------------------------
@when("I call load_files with a nonexistent file path")
def step_load_files_missing(context: Any) -> None:
state = _make_state(file_paths=["/tmp/does_not_exist_xyz.py"])
context.load_result = context.graph_agent._load_files(state)
@then("the returned documents list should be empty")
def step_returned_docs_empty(context: Any) -> None:
assert context.load_result["documents"] == []
@then('the returned error should contain "{text}"')
def step_returned_error_contains(context: Any, text: str) -> None:
error = context.load_result.get("error")
assert error is not None and text in error, (
f"Expected '{text}' in error, got: {error!r}"
)
# ---------------------------------------------------------------------------
# Scenario: Load files rejects directory paths
# Targets lines 238-240
# ---------------------------------------------------------------------------
@given('I have a temporary directory called "{dirname}"')
def step_create_temp_directory(context: Any, dirname: str) -> None:
temp_dir = _ensure_temp_dir(context)
dir_path = temp_dir / dirname
dir_path.mkdir(parents=True, exist_ok=True)
context.graph_dir_path = dir_path
@when("I call load_files with the directory path")
def step_load_files_with_dir(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_dir_path)])
context.load_result = context.graph_agent._load_files(state)
# ---------------------------------------------------------------------------
# Scenario: Load files reads a real file from disk
# Targets lines 242-244 (TextLoader path)
# ---------------------------------------------------------------------------
@given('I have a temporary file "{filename}" containing "{content}"')
def step_create_temp_file(context: Any, filename: str, content: str) -> None:
temp_dir = _ensure_temp_dir(context)
file_path = temp_dir / filename
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_text(content, encoding="utf-8")
context.graph_file_path = file_path
@when("I call load_files with that file path")
def step_load_files_real_file(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_file_path)])
context.load_result = context.graph_agent._load_files(state)
@then("the returned documents list should have {count:d} entry")
def step_returned_docs_count(context: Any, count: int) -> None:
assert len(context.load_result["documents"]) == count
@then("the returned error should be empty")
def step_returned_error_empty(context: Any) -> None:
assert context.load_result.get("error") is None
# ---------------------------------------------------------------------------
# Scenario: Load files combines missing and directory errors
# Targets lines 231-250 comprehensively
# ---------------------------------------------------------------------------
@when("I call load_files with both a missing file and the directory path")
def step_load_files_mixed_errors(context: Any) -> None:
state = _make_state(
file_paths=[
"/tmp/definitely_missing_abc.py",
str(context.graph_dir_path),
]
)
context.load_result = context.graph_agent._load_files(state)
# ---------------------------------------------------------------------------
# Scenario: Chunking splits documents exceeding chunk size
# Targets lines 317-326 (the step/chunking loop)
# ---------------------------------------------------------------------------
@given("I have a state with a single document of {size:d} characters")
def step_state_with_large_doc(context: Any, size: int) -> None:
content = "a" * size
doc = Document(page_content=content, metadata={"source": "large.py"})
context.graph_state = _make_state(documents=[doc])
@when("I call chunk_documents on the state")
def step_call_chunk_documents(context: Any) -> None:
result = context.graph_agent._chunk_documents(context.graph_state)
context.graph_state.update(result)
@then("the resulting chunks list should have at least {count:d} entries")
def step_chunks_at_least(context: Any, count: int) -> None:
assert len(context.graph_state["chunks"]) >= count
@then("every chunk should carry a chunk_index in its metadata")
def step_chunks_have_metadata(context: Any) -> None:
for chunk in context.graph_state["chunks"]:
assert "chunk_index" in chunk.metadata
# ---------------------------------------------------------------------------
# Scenario: Relevance scoring skips duplicate file chunks
# Targets line 345 (if file_path in files_seen: continue)
# ---------------------------------------------------------------------------
@given('I have a state with two chunks from the same source file "{source}"')
def step_state_dup_chunks(context: Any, source: str) -> None:
chunks = [
Document(
page_content="chunk 0 content",
metadata={"source": source, "chunk_index": 0},
),
Document(
page_content="chunk 1 content",
metadata={"source": source, "chunk_index": 1},
),
]
context.graph_state = _make_state(chunks=chunks)
@when("I call score_relevance on the state")
def step_call_score_relevance(context: Any) -> None:
result = context.graph_agent._score_relevance(context.graph_state)
context.graph_state.update(result)
@then("the relevance scores dictionary should contain exactly {count:d} entry")
def step_scores_count(context: Any, count: int) -> None:
assert len(context.graph_state["relevance_scores"]) == count
# ---------------------------------------------------------------------------
# Scenario: Relevance parser returns 0.3 for low keyword
# Targets line 388 (if "low" in llm_output.lower(): return 0.3)
# ---------------------------------------------------------------------------
@when('I parse relevance from the text "{text}"')
def step_parse_relevance(context: Any, text: str) -> None:
context.parsed_relevance = context.graph_agent._parse_relevance_score(text)
@then("the parsed relevance score should be {expected:f}")
def step_parsed_relevance_value(context: Any, expected: float) -> None:
assert abs(context.parsed_relevance - expected) < 1e-9, (
f"Expected {expected}, got {context.parsed_relevance}"
)
# ---------------------------------------------------------------------------
# Scenario: Async invocation completes the full workflow
# Targets lines 444-446 (ainvoke method)
# ---------------------------------------------------------------------------
@when("I run the workflow asynchronously via ainvoke")
def step_ainvoke_workflow(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_file_path)])
config = {"configurable": {"thread_id": "graph-cov-async"}}
async def _run() -> ContextAnalysisState:
return await context.graph_agent.ainvoke(state, config=config)
context.async_result = asyncio.run(_run())
@then("the async result should contain a summary")
def step_async_result_summary(context: Any) -> None:
assert "summary" in context.async_result
assert isinstance(context.async_result["summary"], str)
assert context.async_result["summary"] != ""
@then("the async result should contain documents")
def step_async_result_documents(context: Any) -> None:
assert "documents" in context.async_result
# ---------------------------------------------------------------------------
# Scenario: Sync streaming produces node-level events
# Targets lines 452-458 (stream method)
# ---------------------------------------------------------------------------
@when("I stream the workflow synchronously")
def step_sync_stream(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_file_path)])
config = {"configurable": {"thread_id": "graph-cov-stream"}}
context.stream_events = list(context.graph_agent.stream(state, config=config))
@then("I should receive at least {count:d} stream event")
def step_stream_event_count(context: Any, count: int) -> None:
assert len(context.stream_events) >= count
@then("each stream event should be a dictionary with a node key")
def step_stream_events_are_dicts(context: Any) -> None:
expected_nodes = {
"load_files",
"analyze_dependencies",
"chunk_documents",
"score_relevance",
"summarize_context",
}
for event in context.stream_events:
assert isinstance(event, dict)
assert any(key in expected_nodes for key in event)
# ---------------------------------------------------------------------------
# Scenario: Async streaming produces node-level events
# Targets lines 464-471 (astream method + async for yield)
# ---------------------------------------------------------------------------
@when("I stream the workflow asynchronously")
def step_async_stream(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_file_path)])
config = {"configurable": {"thread_id": "graph-cov-astream"}}
async def _run() -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
async for event in context.graph_agent.astream(state, config=config):
events.append(event)
return events
context.astream_events = asyncio.run(_run())
@then("I should receive at least {count:d} async stream event")
def step_astream_event_count(context: Any, count: int) -> None:
assert len(context.astream_events) >= count
@then("each async stream event should be a dictionary with a node key")
def step_astream_events_are_dicts(context: Any) -> None:
expected_nodes = {
"load_files",
"analyze_dependencies",
"chunk_documents",
"score_relevance",
"summarize_context",
}
for event in context.astream_events:
assert isinstance(event, dict)
assert any(key in expected_nodes for key in event)
# ---------------------------------------------------------------------------
# Scenario: Agent initialises with default FakeListLLM when no LLM provided
# Targets lines 114-117 (if llm is None: FakeListLLM)
# ---------------------------------------------------------------------------
@when("I create a ContextAnalysisAgent without providing an LLM")
def step_create_agent_no_llm(context: Any) -> None:
context.graph_agent = ContextAnalysisAgent(llm=_default_context_test_llm())
@then("the agent LLM should be a FakeListLLM")
def step_agent_llm_is_fake(context: Any) -> None:
assert type(context.graph_agent.llm).__name__ == "FakeListLLM"
# ---------------------------------------------------------------------------
# Scenario: Sync invoke runs the complete workflow end to end
# Targets lines 436-438 (invoke method)
# ---------------------------------------------------------------------------
@when("I invoke the workflow synchronously")
def step_sync_invoke(context: Any) -> None:
state = _make_state(file_paths=[str(context.graph_file_path)])
config = {"configurable": {"thread_id": "graph-cov-invoke"}}
context.sync_result = context.graph_agent.invoke(state, config=config)
@then("the sync result should contain a summary")
def step_sync_result_summary(context: Any) -> None:
assert "summary" in context.sync_result
assert isinstance(context.sync_result["summary"], str)
assert context.sync_result["summary"] != ""
@then("the sync result should contain documents")
def step_sync_result_documents(context: Any) -> None:
assert "documents" in context.sync_result
# ---------------------------------------------------------------------------
# Scenario: Load files returns preloaded documents unchanged
# Targets lines 221-226 (if preloaded_docs: return early)
# ---------------------------------------------------------------------------
@given("I have a state with preloaded documents")
def step_state_with_preloaded_docs(context: Any) -> None:
context.preloaded_docs = [
Document(page_content="preloaded content", metadata={"source": "pre.py"}),
]
context.preloaded_state = _make_state(documents=context.preloaded_docs)
@when("I call load_files on the preloaded state")
def step_load_files_preloaded(context: Any) -> None:
context.load_result = context.graph_agent._load_files(context.preloaded_state)
@then("the returned documents should equal the preloaded documents")
def step_returned_docs_equal_preloaded(context: Any) -> None:
assert context.load_result["documents"] == context.preloaded_docs
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
def after_scenario(context: Any, _scenario: Any) -> None:
"""Clean up temporary files created during graph coverage scenarios."""
temp_dir = getattr(context, "graph_temp_dir", None)
if temp_dir is not None and Path(temp_dir).exists():
shutil.rmtree(temp_dir, ignore_errors=True)
for attr in (
"graph_agent",
"graph_temp_dir",
"graph_file_path",
"graph_dir_path",
"graph_state",
"load_result",
"custom_llm_ref",
"parsed_relevance",
"async_result",
"stream_events",
"astream_events",
):
if hasattr(context, attr):
setattr(context, attr, None)
# ---------------------------------------------------------------------------
# Scenario: Agent raises ValueError when no LLM provided
# ---------------------------------------------------------------------------
@when("I create a context analysis agent with no LLM expecting an error")
def step_create_agent_no_llm_error(context: Any) -> None:
context.raised_error = None
try:
from cleveragents.agents.graphs.context_analysis import (
ContextAnalysisAgent as _CA,
)
_CA(llm=None)
except ValueError as exc:
context.raised_error = exc
@then("a ValueError should have been raised about missing LLM")
def step_check_value_error_missing_llm(context: Any) -> None:
assert context.raised_error is not None, (
"Expected ValueError but no error was raised"
)
assert "No LLM provider configured" in str(context.raised_error)