Files
temp/features/steps/context_analysis_new_coverage_steps.py
freemo a074b4846f fix(provider): remove FakeListLLM defaults
Remove FakeListLLM as a silent fallback in agent graph constructors
(plan_generation.py, context_analysis.py, auto_debug.py). All three now
raise ValueError when llm=None, making missing-provider errors explicit.

Add Settings.mock_providers flag and validate_provider_availability()
method. Update container.get_ai_provider() to check Settings.mock_providers
first, with env-var fallback for backward compatibility.

Add resolve_provider_by_name() helper to the provider registry and export
it from cleveragents.providers. Add structlog trace logging to
ProviderRegistry.get_default_provider_type() to record selection reasoning.

Update all existing behave step files, robot tests, and benchmarks that
relied on the implicit FakeListLLM default to pass an explicit LLM
instance instead.

Add new BDD tests (features/provider_fixes.feature with 17 scenarios),
Robot Framework integration tests (robot/provider_detection_smoke.robot),
and ASV benchmarks (benchmarks/provider_selection_bench.py).

ISSUES CLOSED: #323
2026-02-27 09:47:10 -05:00

302 lines
9.9 KiB
Python

"""Step definitions for context_analysis_new_coverage.feature."""
from __future__ import annotations
import tempfile
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when # type: ignore[import-untyped]
from behave.runner import Context # type: ignore[import-untyped]
from langchain_community.llms import FakeListLLM
from langchain_core.documents import Document
from cleveragents.agents.graphs.context_analysis import (
ContextAnalysisAgent,
ContextAnalysisState,
)
def _default_context_test_llm() -> FakeListLLM:
"""Create a FakeListLLM for context analysis test purposes."""
return FakeListLLM(
responses=[
"Dependencies: ['os', 'sys', 'pathlib']",
"Relevance: High - contains core functionality",
"Summary: Python module implementing core business logic",
]
)
def _empty_state(**overrides: Any) -> ContextAnalysisState:
base: ContextAnalysisState = {
"file_paths": [],
"documents": [],
"dependencies": {},
"summary": "",
"relevance_scores": {},
"chunks": [],
"error": None,
}
base.update(overrides) # type: ignore[typeddict-item]
return base
@when("I create a ContextAnalysisAgent without an LLM")
def step_create_default(context: Context) -> None:
context.agent = ContextAnalysisAgent(llm=_default_context_test_llm())
@when("I create a new ContextAnalysisAgent without an LLM expecting error")
def step_create_no_llm_error(context: Context) -> None:
context.raised_error = None
try:
ContextAnalysisAgent(llm=None)
except ValueError as exc:
context.raised_error = exc
@then("a ValueError should be raised about missing LLM")
def step_assert_value_error_missing_llm(context: Context) -> None:
assert context.raised_error is not None, "Expected ValueError but none was raised"
assert "No LLM provider configured" in str(context.raised_error)
@then("the agent should use FakeListLLM")
def step_assert_fake_llm(context: Context) -> None:
assert isinstance(context.agent.llm, FakeListLLM)
@then("the agent should have a compiled graph")
def step_assert_compiled(context: Context) -> None:
assert context.agent.app is not None
@when("I create a ContextAnalysisAgent with a mock LLM")
def step_create_with_llm(context: Context) -> None:
custom_llm = FakeListLLM(responses=["custom response"])
context.agent = ContextAnalysisAgent(llm=custom_llm)
@then("the agent should use the provided LLM")
def step_assert_custom_llm(context: Context) -> None:
assert isinstance(context.agent.llm, FakeListLLM)
@given("a ContextAnalysisAgent instance")
def step_agent_instance(context: Context) -> None:
context.agent = ContextAnalysisAgent(llm=_default_context_test_llm())
@when("I invoke _load_files with preloaded documents")
def step_load_preloaded(context: Context) -> None:
doc = Document(page_content="hello world", metadata={"source": "test.py"})
state = _empty_state(documents=[doc])
context.result = context.agent._load_files(state)
@then("the result should return those preloaded documents")
def step_assert_preloaded(context: Context) -> None:
assert len(context.result["documents"]) == 1
@given("a temporary Python file on disk")
def step_temp_py_file(context: Context) -> None:
with tempfile.NamedTemporaryFile(
mode="w", suffix=".py", delete=False, encoding="utf-8"
) as tmp:
tmp.write("import os\nimport sys\n\ndef main():\n pass\n")
tmp.flush()
context.temp_file = tmp.name
@when("I invoke _load_files with the temp file path")
def step_load_temp(context: Context) -> None:
state = _empty_state(file_paths=[context.temp_file])
context.result = context.agent._load_files(state)
@then("the result should contain loaded documents")
def step_assert_loaded(context: Context) -> None:
assert len(context.result["documents"]) > 0
@when("I invoke _load_files with a nonexistent file path")
def step_load_missing(context: Context) -> None:
state = _empty_state(file_paths=["/tmp/nonexistent_file_xyz.py"])
context.result = context.agent._load_files(state)
@then("the result error should mention file not found")
def step_assert_file_not_found(context: Context) -> None:
assert context.result["error"] is not None
assert "not found" in context.result["error"].lower()
@when("I invoke _load_files with a directory path")
def step_load_directory(context: Context) -> None:
state = _empty_state(file_paths=["/tmp"])
context.result = context.agent._load_files(state)
@then("the result error should mention not a file")
def step_assert_not_file(context: Context) -> None:
assert context.result["error"] is not None
assert "not a file" in context.result["error"].lower()
@given("a state with one loaded document")
def step_one_doc_state(context: Context) -> None:
doc = Document(
page_content="import os\nimport sys\n", metadata={"source": "test.py"}
)
context.state = _empty_state(documents=[doc])
@when("I invoke _analyze_dependencies")
def step_analyze_deps(context: Context) -> None:
context.result = context.agent._analyze_dependencies(context.state)
@then("the dependencies dict should have entries")
def step_assert_deps(context: Context) -> None:
assert len(context.result["dependencies"]) > 0
@when('I parse dependencies from "{text}"')
def step_parse_deps(context: Context, text: str) -> None:
context.result = context.agent._parse_dependencies(text)
@then("the parsed list should contain os sys pathlib")
def step_assert_parsed_deps(context: Context) -> None:
deps_lower = [d.lower() for d in context.result]
for name in ("os", "sys", "pathlib"):
assert any(name in d for d in deps_lower), (
f"'{name}' not found in {context.result}"
)
@given("a state with a small document")
def step_small_doc(context: Context) -> None:
doc = Document(page_content="short content", metadata={"source": "small.py"})
context.state = _empty_state(documents=[doc])
@when("I invoke _chunk_documents")
def step_chunk_docs(context: Context) -> None:
context.result = context.agent._chunk_documents(context.state)
@then("the chunks should contain the original document")
def step_assert_original_chunk(context: Context) -> None:
assert len(context.result["chunks"]) == 1
@given("a ContextAnalysisAgent instance with chunk_size {size:d}")
def step_agent_custom_chunk(context: Context, size: int) -> None:
context.agent = ContextAnalysisAgent(
llm=_default_context_test_llm(), chunk_size=size, chunk_overlap=20
)
@given("a state with a large document of {n:d} characters")
def step_large_doc(context: Context, n: int) -> None:
doc = Document(page_content="x" * n, metadata={"source": "large.py"})
context.state = _empty_state(documents=[doc])
@then("the chunks should contain more than one document")
def step_assert_multiple_chunks(context: Context) -> None:
assert len(context.result["chunks"]) > 1
@given("a state with chunks from two files")
def step_chunks_two_files(context: Context) -> None:
chunks = [
Document(page_content="code a", metadata={"source": "a.py"}),
Document(page_content="code b", metadata={"source": "b.py"}),
]
context.state = _empty_state(chunks=chunks)
@when("I invoke _score_relevance")
def step_score_relevance(context: Context) -> None:
context.result = context.agent._score_relevance(context.state)
@then("relevance scores should exist for both files")
def step_assert_both_scores(context: Context) -> None:
scores = context.result["relevance_scores"]
assert len(scores) == 2
@when('I parse relevance score from "{text}"')
def step_parse_score(context: Context, text: str) -> None:
context.parsed_score = context.agent._parse_relevance_score(text)
@then("the parsed score should be approximately {expected}")
def step_assert_approx_score(context: Context, expected: str) -> None:
assert abs(context.parsed_score - float(expected)) < 0.1, (
f"Expected ~{expected}, got {context.parsed_score}"
)
@given("a full analysis state with documents and scores")
def step_full_state(context: Context) -> None:
doc = Document(page_content="import os\n", metadata={"source": "main.py"})
context.state = _empty_state(
documents=[doc],
dependencies={"main.py": ["os"]},
relevance_scores={"main.py": 0.9},
chunks=[doc],
)
@when("I invoke _summarize_context")
def step_summarize(context: Context) -> None:
context.result = context.agent._summarize_context(context.state)
@then("the direct summary should be a non-empty string")
def step_assert_summary(context: Context) -> None:
assert context.result["summary"]
@when("I invoke the full workflow with the temp file")
def step_invoke_full(context: Context) -> None:
state = _empty_state(file_paths=[context.temp_file])
config = {"configurable": {"thread_id": "test-invoke"}}
context.result = context.agent.invoke(state, config)
@then("the result should contain summary and scores")
def step_assert_full_result(context: Context) -> None:
assert "summary" in context.result
assert "relevance_scores" in context.result
@when("I stream the workflow with the temp file")
def step_stream(context: Context) -> None:
state = _empty_state(file_paths=[context.temp_file])
config = {"configurable": {"thread_id": "test-stream"}}
context.events = list(context.agent.stream(state, config))
@then("at least one analysis event should be yielded")
def step_assert_events(context: Context) -> None:
assert len(context.events) > 0
@when("I wrap a chain with retry support")
def step_with_retry(context: Context) -> None:
mock_chain = MagicMock()
mock_chain.with_retry.return_value = mock_chain
context.result = context.agent._with_retry(mock_chain)
@then("the wrapped chain should be returned")
def step_assert_wrapped(context: Context) -> None:
assert context.result is not None