Files
temp/features/steps/context_service_coverage_gaps_steps.py
freemo a074b4846f fix(provider): remove FakeListLLM defaults
Remove FakeListLLM as a silent fallback in agent graph constructors
(plan_generation.py, context_analysis.py, auto_debug.py). All three now
raise ValueError when llm=None, making missing-provider errors explicit.

Add Settings.mock_providers flag and validate_provider_availability()
method. Update container.get_ai_provider() to check Settings.mock_providers
first, with env-var fallback for backward compatibility.

Add resolve_provider_by_name() helper to the provider registry and export
it from cleveragents.providers. Add structlog trace logging to
ProviderRegistry.get_default_provider_type() to record selection reasoning.

Update all existing behave step files, robot tests, and benchmarks that
relied on the implicit FakeListLLM default to pass an explicit LLM
instance instead.

Add new BDD tests (features/provider_fixes.feature with 17 scenarios),
Robot Framework integration tests (robot/provider_detection_smoke.robot),
and ASV benchmarks (benchmarks/provider_selection_bench.py).

ISSUES CLOSED: #323
2026-02-27 09:47:10 -05:00

629 lines
20 KiB
Python

"""Step definitions for context_service.py coverage gap tests.
Targets lines identified as uncovered in build/coverage.xml:
- _load_agentsignore exception branch (268-269)
- _get_current_plan (525-528)
- _build_langsmith_config (540-560)
- _prepare_analysis_config (577-585)
- _refresh_vector_index ConfigurationError (595-598)
- _get_context_agent (624-626)
- analyze_context (656-691)
- analyze_context_async (709-742)
- analyze_context_streaming (761-787)
- analyze_context_streaming_async (805-832)
- get_context_summary (851-852)
- get_context_dependencies (871-872)
- get_relevant_files (892-898)
- search_context (910-944)
"""
from __future__ import annotations
import asyncio
import contextlib
import os
import shutil
import stat
import tempfile
from pathlib import Path
from types import MethodType
from typing import Any
from unittest.mock import patch
from behave import given, then, when
from cleveragents.application.services.context_service import ContextService
from cleveragents.config.settings import Settings
from cleveragents.core.exceptions import ConfigurationError
from cleveragents.domain.models.core import Plan, Project
from features.steps.service_steps import add_cleanup
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class _InMemoryContextRepo:
"""Minimal in-memory context repository."""
def __init__(self) -> None:
self._store: dict[int, list] = {}
self._next_id = 1
def get_for_plan(self, plan_id: int) -> list:
return list(self._store.get(plan_id, []))
def add(self, context) -> None:
context.id = context.id or self._next_id
self._next_id += 1
self._store.setdefault(context.plan_id, []).append(context)
def remove(self, context_id: int) -> None:
for pid in self._store:
self._store[pid] = [c for c in self._store[pid] if c.id != context_id]
def clear_for_plan(self, plan_id: int) -> None:
self._store.pop(plan_id, None)
class _InMemoryPlansRepo:
def __init__(self, plan: Plan | None) -> None:
self._plan = plan
def get_current_for_project(self, project_id: int | None) -> Plan | None:
return self._plan if project_id else None
class _InMemoryTransaction:
def __init__(self, plans: _InMemoryPlansRepo, contexts: _InMemoryContextRepo):
self.plans = plans
self.contexts = contexts
def __enter__(self):
return self
def __exit__(self, *args):
return False
class _InMemoryUnitOfWork:
def __init__(self, plan: Plan | None) -> None:
self._plans = _InMemoryPlansRepo(plan)
self._contexts = _InMemoryContextRepo()
def transaction(self):
return _InMemoryTransaction(self._plans, self._contexts)
class _VectorStoreStub:
"""Configurable vector store test double."""
def __init__(self, *, enabled: bool) -> None:
self.enabled = enabled
self.refresh_calls: list[int] = []
self.search_calls: list[tuple] = []
self.results: list[dict[str, Any]] = []
self.refresh_error: Exception | None = None
self.search_error: Exception | None = None
def is_enabled(self) -> bool:
return self.enabled
def refresh_for_plan(self, plan_id: int) -> int:
self.refresh_calls.append(plan_id)
if self.refresh_error:
raise self.refresh_error
return len(self.results)
def search(
self,
plan_id: int,
query: str,
*,
top_k: int = 5,
refresh_if_missing: bool = True,
) -> list[dict[str, Any]]:
self.search_calls.append((plan_id, query, top_k, refresh_if_missing))
if self.search_error:
raise self.search_error
return list(self.results)
def _make_workspace(context, *, vector_stub=None):
"""Build a coverage gap workspace with in-memory UoW."""
temp_dir = Path(tempfile.mkdtemp(prefix="cov-gap-"))
add_cleanup(context, lambda: shutil.rmtree(temp_dir, ignore_errors=True))
plan = Plan(
id=201,
project_id=1,
name="cov-plan",
prompt="coverage",
current=True,
)
project = Project(id=1, name="cov-project", path=temp_dir)
settings = Settings()
uow = _InMemoryUnitOfWork(plan)
service = ContextService(
settings=settings,
unit_of_work=uow,
vector_store_service=vector_stub,
)
context.temp_dir = temp_dir
context.cov_plan = plan
context.cov_project = project
context.cov_settings = settings
context.cov_uow = uow
context.cov_service = service
context.cov_context_files = []
# Stub list_files so the analysis methods get predictable file paths
def _list_files_stub(self, project=None):
return list(context.cov_context_files)
service.list_files = MethodType(_list_files_stub, service)
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given("a coverage gap workspace")
def step_coverage_gap_workspace(context):
_make_workspace(context)
@given("an unreadable .agentsignore file in the workspace")
def step_unreadable_agentsignore(context):
ignore_file = context.temp_dir / ".agentsignore"
ignore_file.write_text("pattern_to_ignore\n")
# Remove read permission to trigger the except branch
ignore_file.chmod(0o000)
original_read_text = Path.read_text
def _read_text(path_self: Path, *args, **kwargs):
if path_self == ignore_file:
raise PermissionError("Simulated unreadable .agentsignore")
return original_read_text(path_self, *args, **kwargs)
patcher = patch.object(Path, "read_text", _read_text)
patcher.start()
add_cleanup(context, patcher.stop)
def restore():
with contextlib.suppress(Exception):
ignore_file.chmod(stat.S_IRUSR | stat.S_IWUSR)
add_cleanup(context, restore)
@given("a coverage gap workspace with LangSmith tracing enabled")
def step_workspace_langsmith_enabled(context):
_make_workspace(context)
context.cov_settings.langsmith_enabled = True
context.cov_settings.langsmith_api_key = "ls-fake-key-for-coverage"
context.cov_settings.langsmith_project = "cov-test-project"
# Also set environment variables that Settings._evaluate_langsmith_configuration needs
os.environ["LANGCHAIN_API_KEY"] = "ls-fake-key-for-coverage"
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_PROJECT"] = "cov-test-project"
def cleanup_env():
os.environ.pop("LANGCHAIN_API_KEY", None)
os.environ.pop("LANGCHAIN_TRACING_V2", None)
os.environ.pop("LANGCHAIN_PROJECT", None)
add_cleanup(context, cleanup_env)
@given("a coverage gap workspace with a failing vector store")
def step_workspace_failing_vector_store(context):
stub = _VectorStoreStub(enabled=True)
stub.refresh_error = ConfigurationError("Simulated config error for coverage")
_make_workspace(context, vector_stub=stub)
context.cov_vector_stub = stub
@given("a coverage gap workspace with no context files")
def step_workspace_no_context_files(context):
_make_workspace(context)
context.cov_context_files = []
@given('a coverage gap workspace with a context file "{filename}"')
def step_workspace_with_context_file(context, filename: str):
_make_workspace(context)
file_path = context.temp_dir / filename
file_path.write_text(
"import os\nimport sys\n\ndef hello():\n print('hello')\n",
encoding="utf-8",
)
context.cov_context_files = [str(file_path)]
@given("a coverage gap workspace with vector search enabled")
def step_workspace_vector_enabled(context):
stub = _VectorStoreStub(enabled=True)
_make_workspace(context, vector_stub=stub)
context.cov_vector_stub = stub
@given("a coverage gap workspace with vector search disabled")
def step_workspace_vector_disabled(context):
stub = _VectorStoreStub(enabled=False)
_make_workspace(context, vector_stub=stub)
context.cov_vector_stub = stub
@given("the coverage workspace project has no id")
def step_coverage_project_no_id(context):
context.cov_project.id = None
@given("the vector store stub has results for the coverage workspace")
def step_vector_stub_has_results(context):
context.cov_vector_stub.results = [
{"path": "doc.py", "score": 0.95, "snippet": "match"}
]
@given("the vector store search raises a ConfigurationError")
def step_vector_search_config_error(context):
context.cov_vector_stub.search_error = ConfigurationError(
"Simulated search config error"
)
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when("I load agentsignore patterns for the workspace directory")
def step_load_agentsignore(context):
# Clear cache so the load is fresh
context.cov_service._agentsignore_cache.clear()
context.loaded_patterns = context.cov_service._load_agentsignore(context.temp_dir)
@when("I call _get_current_plan with a project that has no id")
def step_get_current_plan_no_id(context):
project_no_id = Project(id=None, name="no-id-project", path=context.temp_dir)
context.current_plan_result = context.cov_service._get_current_plan(project_no_id)
@when("I call _get_current_plan with the workspace project")
def step_get_current_plan_with_id(context):
context.current_plan_result = context.cov_service._get_current_plan(
context.cov_project
)
@when("I build a LangSmith config with tracing disabled")
def step_build_langsmith_disabled(context):
context.langsmith_result = context.cov_service._build_langsmith_config(
context.cov_project,
run_name="DisabledRun",
file_paths=["a.py"],
mode="sync",
)
@when("I build a LangSmith config with tracing enabled")
def step_build_langsmith_enabled(context):
context.langsmith_result = context.cov_service._build_langsmith_config(
context.cov_project,
run_name="EnabledRun",
file_paths=["a.py"],
mode="sync",
)
@when("I prepare an analysis config for the workspace project")
def step_prepare_analysis_config(context):
context.analysis_config = context.cov_service._prepare_analysis_config(
context.cov_project,
run_name="TestRun",
file_paths=["file.py"],
mode="sync",
)
@when("I trigger a vector index refresh for the workspace plan")
def step_trigger_vector_refresh(context):
context.refresh_exception = None
try:
context.cov_service._refresh_vector_index(context.cov_plan.id)
except Exception as exc:
context.refresh_exception = exc
@when("I request a context analysis agent from the coverage service")
def step_request_context_agent(context):
from langchain_community.llms import FakeListLLM
context.cov_agent = context.cov_service._get_context_agent(
llm=FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
),
)
@when("I run analyze_context on the workspace project")
def step_run_analyze_context(context):
from langchain_community.llms import FakeListLLM
context.analysis_result = context.cov_service.analyze_context(
context.cov_project,
llm=FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
),
)
@when("I run analyze_context_async on the workspace project")
def step_run_analyze_context_async(context):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
context.async_analysis_result = asyncio.run(
context.cov_service.analyze_context_async(context.cov_project, llm=_fake_llm)
)
@when("I stream analyze_context on the workspace project")
def step_stream_analyze_context(context):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
context.streaming_events = list(
context.cov_service.analyze_context_streaming(
context.cov_project, llm=_fake_llm
)
)
@when("I async stream analyze_context on the workspace project")
def step_async_stream_analyze_context(context):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
async def _collect():
events = []
async for event in context.cov_service.analyze_context_streaming_async(
context.cov_project, llm=_fake_llm
):
events.append(event)
return events
context.async_streaming_events = asyncio.run(_collect())
@when("I get the context summary from the coverage service")
def step_get_context_summary(context):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
context.cov_summary = context.cov_service.get_context_summary(
context.cov_project, llm=_fake_llm
)
@when("I get the context dependencies from the coverage service")
def step_get_context_dependencies(context):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
context.cov_dependencies = context.cov_service.get_context_dependencies(
context.cov_project, llm=_fake_llm
)
@when("I get relevant files with threshold {threshold:f} from the coverage service")
def step_get_relevant_files(context, threshold: float):
from langchain_community.llms import FakeListLLM
_fake_llm = FakeListLLM(
responses=["Dependencies: ['os']", "Relevance: High", "Summary: test"] * 3
)
context.cov_relevant = context.cov_service.get_relevant_files(
context.cov_project, threshold=threshold, llm=_fake_llm
)
@when('I search the coverage context for "{query}"')
def step_search_coverage_context(context, query: str):
context.cov_search_results = context.cov_service.search_context(
context.cov_project, query
)
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then("the loaded patterns should be an empty list")
def step_patterns_empty(context):
assert context.loaded_patterns == [], (
f"Expected empty list, got {context.loaded_patterns}"
)
@then("the coverage _get_current_plan result should be None")
def step_cov_result_none(context):
assert context.current_plan_result is None, (
f"Expected None, got {context.current_plan_result}"
)
@then("the coverage _get_current_plan result should match the workspace plan")
def step_cov_result_current_plan(context):
assert context.current_plan_result is not None, "Expected a plan, got None"
assert context.current_plan_result.id == context.cov_plan.id, (
f"Expected plan id {context.cov_plan.id}, got {context.current_plan_result.id}"
)
@then("the LangSmith config result should be an empty dict")
def step_langsmith_empty(context):
assert context.langsmith_result == {}, (
f"Expected empty dict, got {context.langsmith_result}"
)
@then("the LangSmith config result should contain project metadata")
def step_langsmith_has_metadata(context):
result = context.langsmith_result
assert "metadata" in result, f"Missing metadata key: {result}"
metadata = result["metadata"]
assert metadata.get("project_id") == context.cov_project.id, metadata
assert metadata.get("service") == "ContextService", metadata
@then("the LangSmith config result should contain a project tag")
def step_langsmith_has_project_tag(context):
result = context.langsmith_result
tags = result.get("tags", [])
assert any(t.startswith("project:") for t in tags), (
f"No project tag found in {tags}"
)
@then("the analysis config should contain a configurable thread_id")
def step_config_has_thread_id(context):
configurable = context.analysis_config.get("configurable", {})
thread_id = configurable.get("thread_id", "")
assert thread_id.startswith("context-analysis-"), (
f"Unexpected thread_id: {thread_id}"
)
@then("the refresh should complete without raising an exception")
def step_refresh_no_exception(context):
assert context.refresh_exception is None, (
f"Unexpected exception: {context.refresh_exception}"
)
@then("the returned agent should be a ContextAnalysisAgent instance")
def step_agent_is_correct_type(context):
from cleveragents.agents.context_analysis import ContextAnalysisAgent
assert isinstance(context.cov_agent, ContextAnalysisAgent), (
f"Expected ContextAnalysisAgent, got {type(context.cov_agent)}"
)
@then("the analysis documents should be empty")
def step_analysis_docs_empty(context):
docs = context.analysis_result.get("documents", [])
assert len(docs) == 0, f"Expected 0 documents, got {len(docs)}"
@then("the analysis summary should say no files to analyze")
def step_analysis_summary_no_files(context):
summary = context.analysis_result.get("summary", "")
assert "no context files" in summary.lower(), f"Unexpected summary: {summary}"
@then("the analysis documents should not be empty")
def step_analysis_docs_not_empty(context):
docs = context.analysis_result.get("documents", [])
assert len(docs) > 0, "Expected at least 1 document"
@then("the async analysis documents should be empty")
def step_async_docs_empty(context):
docs = context.async_analysis_result.get("documents", [])
assert len(docs) == 0, f"Expected 0 async documents, got {len(docs)}"
@then("the async analysis summary should say no files to analyze")
def step_async_summary_no_files(context):
summary = context.async_analysis_result.get("summary", "")
assert "no context files" in summary.lower(), f"Unexpected async summary: {summary}"
@then("the async analysis documents should not be empty")
def step_async_docs_not_empty(context):
docs = context.async_analysis_result.get("documents", [])
assert len(docs) > 0, "Expected at least 1 async document"
@then("the only streaming event should indicate no files to analyze")
def step_streaming_no_files(context):
assert context.streaming_events == [
{"type": "complete", "summary": "No context files to analyze"}
], f"Unexpected streaming events: {context.streaming_events}"
@then("the streaming events should include workflow node results")
def step_streaming_has_nodes(context):
assert len(context.streaming_events) >= 1, (
f"Expected at least 1 event, got {len(context.streaming_events)}"
)
@then("the only async streaming event should indicate no files to analyze")
def step_async_streaming_no_files(context):
assert context.async_streaming_events == [
{"type": "complete", "summary": "No context files to analyze"}
], f"Unexpected async streaming events: {context.async_streaming_events}"
@then("the async streaming events should contain workflow results")
def step_async_streaming_has_events(context):
assert len(context.async_streaming_events) >= 1, (
f"Expected at least 1 async event, got {len(context.async_streaming_events)}"
)
@then("the summary should be a non-empty string from coverage service")
def step_summary_not_empty(context):
assert context.cov_summary and context.cov_summary.strip(), (
f"Summary is empty: {context.cov_summary!r}"
)
@then("the dependency result should be a dictionary")
def step_dependencies_is_dict(context):
assert isinstance(context.cov_dependencies, dict), (
f"Expected dict, got {type(context.cov_dependencies)}"
)
@then("the relevant files result should be a list of tuples")
def step_relevant_is_list(context):
assert isinstance(context.cov_relevant, list), (
f"Expected list, got {type(context.cov_relevant)}"
)
@then("the coverage search results should be empty")
def step_search_empty(context):
assert context.cov_search_results == [], (
f"Expected empty results, got {context.cov_search_results}"
)
@then("the coverage search results should not be empty")
def step_search_not_empty(context):
assert len(context.cov_search_results) > 0, "Expected non-empty search results"