forked from HAL9000/cleveragents-core
486 lines
17 KiB
Python
486 lines
17 KiB
Python
"""Step definitions for context analysis graph coverage scenarios.
|
|
|
|
These steps specifically target uncovered lines in
|
|
src/cleveragents/agents/graphs/context_analysis.py as identified
|
|
by the build/coverage.xml report.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from behave import given, then, when
|
|
from langchain_community.llms import FakeListLLM
|
|
from langchain_core.documents import Document
|
|
|
|
from cleveragents.agents.graphs.context_analysis import (
|
|
ContextAnalysisAgent,
|
|
ContextAnalysisState,
|
|
)
|
|
|
|
# Enough responses so every LLM call across the full workflow is satisfied.
|
|
_DEFAULT_RESPONSES = [
|
|
"Dependencies: ['os', 'sys', 'pathlib']",
|
|
"Relevance: 0.8 - core module",
|
|
"Summary: Generated context overview",
|
|
] * 30
|
|
|
|
|
|
def _make_state(**overrides: Any) -> ContextAnalysisState:
|
|
"""Build a minimal valid ContextAnalysisState, with optional overrides."""
|
|
state: ContextAnalysisState = {
|
|
"file_paths": [],
|
|
"documents": [],
|
|
"dependencies": {},
|
|
"summary": "",
|
|
"relevance_scores": {},
|
|
"chunks": [],
|
|
"error": None,
|
|
}
|
|
state.update(overrides) # type: ignore[typeddict-item]
|
|
return state
|
|
|
|
|
|
def _ensure_temp_dir(context: Any) -> Path:
|
|
if not hasattr(context, "graph_temp_dir") or context.graph_temp_dir is None:
|
|
context.graph_temp_dir = Path(tempfile.mkdtemp(prefix="ctx-graph-cov-"))
|
|
return context.graph_temp_dir
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("the context analysis graph module is importable")
|
|
def step_module_importable(context: Any) -> None:
|
|
ContextAnalysisAgent # noqa: B018
|
|
ContextAnalysisState # noqa: B018
|
|
|
|
|
|
@given("I have a fake LLM configured for context analysis graph tests")
|
|
def step_configure_fake_llm(context: Any) -> None:
|
|
context.graph_fake_llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Agent initialises with an explicitly provided LLM
|
|
# Targets lines 115, 117, 125 (the else branch of if llm is None)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I create a ContextAnalysisAgent with a custom LLM")
|
|
def step_create_agent_custom_llm(context: Any) -> None:
|
|
custom_llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
|
|
context.custom_llm_ref = custom_llm
|
|
context.graph_agent = ContextAnalysisAgent(llm=custom_llm)
|
|
|
|
|
|
@then("the agent should use the provided LLM instance")
|
|
def step_agent_uses_provided_llm(context: Any) -> None:
|
|
assert context.graph_agent.llm is context.custom_llm_ref
|
|
|
|
|
|
@then("the agent should be initialised successfully")
|
|
def step_agent_initialised(context: Any) -> None:
|
|
assert isinstance(context.graph_agent, ContextAnalysisAgent)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper: ensure an agent on the context
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _ensure_agent(context: Any, **kwargs: Any) -> ContextAnalysisAgent:
|
|
if hasattr(context, "graph_agent") and context.graph_agent is not None:
|
|
return context.graph_agent
|
|
llm = kwargs.pop("llm", getattr(context, "graph_fake_llm", None))
|
|
if llm is None:
|
|
llm = FakeListLLM(responses=list(_DEFAULT_RESPONSES))
|
|
context.graph_agent = ContextAnalysisAgent(llm=llm, **kwargs)
|
|
return context.graph_agent
|
|
|
|
|
|
@given("I have a ContextAnalysisAgent for graph coverage")
|
|
def step_have_agent(context: Any) -> None:
|
|
context.graph_agent = None # force fresh agent
|
|
_ensure_agent(context)
|
|
|
|
|
|
@given(
|
|
"I have a ContextAnalysisAgent for graph coverage "
|
|
"with chunk_size {chunk_size:d} and chunk_overlap {overlap:d}"
|
|
)
|
|
def step_have_agent_custom_chunks(context: Any, chunk_size: int, overlap: int) -> None:
|
|
context.graph_agent = None
|
|
_ensure_agent(context, chunk_size=chunk_size, chunk_overlap=overlap)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Load files discovers missing file paths
|
|
# Targets lines 228-236, 248-252
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I call load_files with a nonexistent file path")
|
|
def step_load_files_missing(context: Any) -> None:
|
|
state = _make_state(file_paths=["/tmp/does_not_exist_xyz.py"])
|
|
context.load_result = context.graph_agent._load_files(state)
|
|
|
|
|
|
@then("the returned documents list should be empty")
|
|
def step_returned_docs_empty(context: Any) -> None:
|
|
assert context.load_result["documents"] == []
|
|
|
|
|
|
@then('the returned error should contain "{text}"')
|
|
def step_returned_error_contains(context: Any, text: str) -> None:
|
|
error = context.load_result.get("error")
|
|
assert error is not None and text in error, (
|
|
f"Expected '{text}' in error, got: {error!r}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Load files rejects directory paths
|
|
# Targets lines 238-240
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I have a temporary directory called "{dirname}"')
|
|
def step_create_temp_directory(context: Any, dirname: str) -> None:
|
|
temp_dir = _ensure_temp_dir(context)
|
|
dir_path = temp_dir / dirname
|
|
dir_path.mkdir(parents=True, exist_ok=True)
|
|
context.graph_dir_path = dir_path
|
|
|
|
|
|
@when("I call load_files with the directory path")
|
|
def step_load_files_with_dir(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_dir_path)])
|
|
context.load_result = context.graph_agent._load_files(state)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Load files reads a real file from disk
|
|
# Targets lines 242-244 (TextLoader path)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I have a temporary file "{filename}" containing "{content}"')
|
|
def step_create_temp_file(context: Any, filename: str, content: str) -> None:
|
|
temp_dir = _ensure_temp_dir(context)
|
|
file_path = temp_dir / filename
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
file_path.write_text(content, encoding="utf-8")
|
|
context.graph_file_path = file_path
|
|
|
|
|
|
@when("I call load_files with that file path")
|
|
def step_load_files_real_file(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_file_path)])
|
|
context.load_result = context.graph_agent._load_files(state)
|
|
|
|
|
|
@then("the returned documents list should have {count:d} entry")
|
|
def step_returned_docs_count(context: Any, count: int) -> None:
|
|
assert len(context.load_result["documents"]) == count
|
|
|
|
|
|
@then("the returned error should be empty")
|
|
def step_returned_error_empty(context: Any) -> None:
|
|
assert context.load_result.get("error") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Load files combines missing and directory errors
|
|
# Targets lines 231-250 comprehensively
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I call load_files with both a missing file and the directory path")
|
|
def step_load_files_mixed_errors(context: Any) -> None:
|
|
state = _make_state(
|
|
file_paths=[
|
|
"/tmp/definitely_missing_abc.py",
|
|
str(context.graph_dir_path),
|
|
]
|
|
)
|
|
context.load_result = context.graph_agent._load_files(state)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Chunking splits documents exceeding chunk size
|
|
# Targets lines 317-326 (the step/chunking loop)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a state with a single document of {size:d} characters")
|
|
def step_state_with_large_doc(context: Any, size: int) -> None:
|
|
content = "a" * size
|
|
doc = Document(page_content=content, metadata={"source": "large.py"})
|
|
context.graph_state = _make_state(documents=[doc])
|
|
|
|
|
|
@when("I call chunk_documents on the state")
|
|
def step_call_chunk_documents(context: Any) -> None:
|
|
result = context.graph_agent._chunk_documents(context.graph_state)
|
|
context.graph_state.update(result)
|
|
|
|
|
|
@then("the resulting chunks list should have at least {count:d} entries")
|
|
def step_chunks_at_least(context: Any, count: int) -> None:
|
|
assert len(context.graph_state["chunks"]) >= count
|
|
|
|
|
|
@then("every chunk should carry a chunk_index in its metadata")
|
|
def step_chunks_have_metadata(context: Any) -> None:
|
|
for chunk in context.graph_state["chunks"]:
|
|
assert "chunk_index" in chunk.metadata
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Relevance scoring skips duplicate file chunks
|
|
# Targets line 345 (if file_path in files_seen: continue)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('I have a state with two chunks from the same source file "{source}"')
|
|
def step_state_dup_chunks(context: Any, source: str) -> None:
|
|
chunks = [
|
|
Document(
|
|
page_content="chunk 0 content",
|
|
metadata={"source": source, "chunk_index": 0},
|
|
),
|
|
Document(
|
|
page_content="chunk 1 content",
|
|
metadata={"source": source, "chunk_index": 1},
|
|
),
|
|
]
|
|
context.graph_state = _make_state(chunks=chunks)
|
|
|
|
|
|
@when("I call score_relevance on the state")
|
|
def step_call_score_relevance(context: Any) -> None:
|
|
result = context.graph_agent._score_relevance(context.graph_state)
|
|
context.graph_state.update(result)
|
|
|
|
|
|
@then("the relevance scores dictionary should contain exactly {count:d} entry")
|
|
def step_scores_count(context: Any, count: int) -> None:
|
|
assert len(context.graph_state["relevance_scores"]) == count
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Relevance parser returns 0.3 for low keyword
|
|
# Targets line 388 (if "low" in llm_output.lower(): return 0.3)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I parse relevance from the text "{text}"')
|
|
def step_parse_relevance(context: Any, text: str) -> None:
|
|
context.parsed_relevance = context.graph_agent._parse_relevance_score(text)
|
|
|
|
|
|
@then("the parsed relevance score should be {expected:f}")
|
|
def step_parsed_relevance_value(context: Any, expected: float) -> None:
|
|
assert abs(context.parsed_relevance - expected) < 1e-9, (
|
|
f"Expected {expected}, got {context.parsed_relevance}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Async invocation completes the full workflow
|
|
# Targets lines 444-446 (ainvoke method)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I run the workflow asynchronously via ainvoke")
|
|
def step_ainvoke_workflow(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_file_path)])
|
|
config = {"configurable": {"thread_id": "graph-cov-async"}}
|
|
|
|
async def _run() -> ContextAnalysisState:
|
|
return await context.graph_agent.ainvoke(state, config=config)
|
|
|
|
context.async_result = asyncio.run(_run())
|
|
|
|
|
|
@then("the async result should contain a summary")
|
|
def step_async_result_summary(context: Any) -> None:
|
|
assert "summary" in context.async_result
|
|
assert isinstance(context.async_result["summary"], str)
|
|
assert context.async_result["summary"] != ""
|
|
|
|
|
|
@then("the async result should contain documents")
|
|
def step_async_result_documents(context: Any) -> None:
|
|
assert "documents" in context.async_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Sync streaming produces node-level events
|
|
# Targets lines 452-458 (stream method)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I stream the workflow synchronously")
|
|
def step_sync_stream(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_file_path)])
|
|
config = {"configurable": {"thread_id": "graph-cov-stream"}}
|
|
context.stream_events = list(context.graph_agent.stream(state, config=config))
|
|
|
|
|
|
@then("I should receive at least {count:d} stream event")
|
|
def step_stream_event_count(context: Any, count: int) -> None:
|
|
assert len(context.stream_events) >= count
|
|
|
|
|
|
@then("each stream event should be a dictionary with a node key")
|
|
def step_stream_events_are_dicts(context: Any) -> None:
|
|
expected_nodes = {
|
|
"load_files",
|
|
"analyze_dependencies",
|
|
"chunk_documents",
|
|
"score_relevance",
|
|
"summarize_context",
|
|
}
|
|
for event in context.stream_events:
|
|
assert isinstance(event, dict)
|
|
assert any(key in expected_nodes for key in event)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Async streaming produces node-level events
|
|
# Targets lines 464-471 (astream method + async for yield)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I stream the workflow asynchronously")
|
|
def step_async_stream(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_file_path)])
|
|
config = {"configurable": {"thread_id": "graph-cov-astream"}}
|
|
|
|
async def _run() -> list[dict[str, Any]]:
|
|
events: list[dict[str, Any]] = []
|
|
async for event in context.graph_agent.astream(state, config=config):
|
|
events.append(event)
|
|
return events
|
|
|
|
context.astream_events = asyncio.run(_run())
|
|
|
|
|
|
@then("I should receive at least {count:d} async stream event")
|
|
def step_astream_event_count(context: Any, count: int) -> None:
|
|
assert len(context.astream_events) >= count
|
|
|
|
|
|
@then("each async stream event should be a dictionary with a node key")
|
|
def step_astream_events_are_dicts(context: Any) -> None:
|
|
expected_nodes = {
|
|
"load_files",
|
|
"analyze_dependencies",
|
|
"chunk_documents",
|
|
"score_relevance",
|
|
"summarize_context",
|
|
}
|
|
for event in context.astream_events:
|
|
assert isinstance(event, dict)
|
|
assert any(key in expected_nodes for key in event)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Agent initialises with default FakeListLLM when no LLM provided
|
|
# Targets lines 114-117 (if llm is None: FakeListLLM)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I create a ContextAnalysisAgent without providing an LLM")
|
|
def step_create_agent_no_llm(context: Any) -> None:
|
|
context.graph_agent = ContextAnalysisAgent()
|
|
|
|
|
|
@then("the agent LLM should be a FakeListLLM")
|
|
def step_agent_llm_is_fake(context: Any) -> None:
|
|
assert type(context.graph_agent.llm).__name__ == "FakeListLLM"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Sync invoke runs the complete workflow end to end
|
|
# Targets lines 436-438 (invoke method)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I invoke the workflow synchronously")
|
|
def step_sync_invoke(context: Any) -> None:
|
|
state = _make_state(file_paths=[str(context.graph_file_path)])
|
|
config = {"configurable": {"thread_id": "graph-cov-invoke"}}
|
|
context.sync_result = context.graph_agent.invoke(state, config=config)
|
|
|
|
|
|
@then("the sync result should contain a summary")
|
|
def step_sync_result_summary(context: Any) -> None:
|
|
assert "summary" in context.sync_result
|
|
assert isinstance(context.sync_result["summary"], str)
|
|
assert context.sync_result["summary"] != ""
|
|
|
|
|
|
@then("the sync result should contain documents")
|
|
def step_sync_result_documents(context: Any) -> None:
|
|
assert "documents" in context.sync_result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenario: Load files returns preloaded documents unchanged
|
|
# Targets lines 221-226 (if preloaded_docs: return early)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("I have a state with preloaded documents")
|
|
def step_state_with_preloaded_docs(context: Any) -> None:
|
|
context.preloaded_docs = [
|
|
Document(page_content="preloaded content", metadata={"source": "pre.py"}),
|
|
]
|
|
context.preloaded_state = _make_state(documents=context.preloaded_docs)
|
|
|
|
|
|
@when("I call load_files on the preloaded state")
|
|
def step_load_files_preloaded(context: Any) -> None:
|
|
context.load_result = context.graph_agent._load_files(context.preloaded_state)
|
|
|
|
|
|
@then("the returned documents should equal the preloaded documents")
|
|
def step_returned_docs_equal_preloaded(context: Any) -> None:
|
|
assert context.load_result["documents"] == context.preloaded_docs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def after_scenario(context: Any, _scenario: Any) -> None:
|
|
"""Clean up temporary files created during graph coverage scenarios."""
|
|
temp_dir = getattr(context, "graph_temp_dir", None)
|
|
if temp_dir is not None and Path(temp_dir).exists():
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
for attr in (
|
|
"graph_agent",
|
|
"graph_temp_dir",
|
|
"graph_file_path",
|
|
"graph_dir_path",
|
|
"graph_state",
|
|
"load_result",
|
|
"custom_llm_ref",
|
|
"parsed_relevance",
|
|
"async_result",
|
|
"stream_events",
|
|
"astream_events",
|
|
):
|
|
if hasattr(context, attr):
|
|
setattr(context, attr, None)
|