forked from HAL9000/cleveragents-core
brent.edwards
af67476b99
Simulate PermissionError on .agentsignore reads so the _load_agentsignore exception path is exercised even when CI runs as root.
581 lines
19 KiB
Python
581 lines
19 KiB
Python
"""Step definitions for context_service.py coverage gap tests.
|
|
|
|
Targets lines identified as uncovered in build/coverage.xml:
|
|
- _load_agentsignore exception branch (268-269)
|
|
- _get_current_plan (525-528)
|
|
- _build_langsmith_config (540-560)
|
|
- _prepare_analysis_config (577-585)
|
|
- _refresh_vector_index ConfigurationError (595-598)
|
|
- _get_context_agent (624-626)
|
|
- analyze_context (656-691)
|
|
- analyze_context_async (709-742)
|
|
- analyze_context_streaming (761-787)
|
|
- analyze_context_streaming_async (805-832)
|
|
- get_context_summary (851-852)
|
|
- get_context_dependencies (871-872)
|
|
- get_relevant_files (892-898)
|
|
- search_context (910-944)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import tempfile
|
|
from pathlib import Path
|
|
from types import MethodType
|
|
from typing import Any
|
|
from unittest.mock import patch
|
|
|
|
from behave import given, then, when
|
|
|
|
from cleveragents.application.services.context_service import ContextService
|
|
from cleveragents.config.settings import Settings
|
|
from cleveragents.core.exceptions import ConfigurationError
|
|
from cleveragents.domain.models.core import Plan, Project
|
|
from features.steps.service_steps import add_cleanup
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class _InMemoryContextRepo:
|
|
"""Minimal in-memory context repository."""
|
|
|
|
def __init__(self) -> None:
|
|
self._store: dict[int, list] = {}
|
|
self._next_id = 1
|
|
|
|
def get_for_plan(self, plan_id: int) -> list:
|
|
return list(self._store.get(plan_id, []))
|
|
|
|
def add(self, context) -> None:
|
|
context.id = context.id or self._next_id
|
|
self._next_id += 1
|
|
self._store.setdefault(context.plan_id, []).append(context)
|
|
|
|
def remove(self, context_id: int) -> None:
|
|
for pid in self._store:
|
|
self._store[pid] = [c for c in self._store[pid] if c.id != context_id]
|
|
|
|
def clear_for_plan(self, plan_id: int) -> None:
|
|
self._store.pop(plan_id, None)
|
|
|
|
|
|
class _InMemoryPlansRepo:
|
|
def __init__(self, plan: Plan | None) -> None:
|
|
self._plan = plan
|
|
|
|
def get_current_for_project(self, project_id: int | None) -> Plan | None:
|
|
return self._plan if project_id else None
|
|
|
|
|
|
class _InMemoryTransaction:
|
|
def __init__(self, plans: _InMemoryPlansRepo, contexts: _InMemoryContextRepo):
|
|
self.plans = plans
|
|
self.contexts = contexts
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
|
|
class _InMemoryUnitOfWork:
|
|
def __init__(self, plan: Plan | None) -> None:
|
|
self._plans = _InMemoryPlansRepo(plan)
|
|
self._contexts = _InMemoryContextRepo()
|
|
|
|
def transaction(self):
|
|
return _InMemoryTransaction(self._plans, self._contexts)
|
|
|
|
|
|
class _VectorStoreStub:
|
|
"""Configurable vector store test double."""
|
|
|
|
def __init__(self, *, enabled: bool) -> None:
|
|
self.enabled = enabled
|
|
self.refresh_calls: list[int] = []
|
|
self.search_calls: list[tuple] = []
|
|
self.results: list[dict[str, Any]] = []
|
|
self.refresh_error: Exception | None = None
|
|
self.search_error: Exception | None = None
|
|
|
|
def is_enabled(self) -> bool:
|
|
return self.enabled
|
|
|
|
def refresh_for_plan(self, plan_id: int) -> int:
|
|
self.refresh_calls.append(plan_id)
|
|
if self.refresh_error:
|
|
raise self.refresh_error
|
|
return len(self.results)
|
|
|
|
def search(
|
|
self,
|
|
plan_id: int,
|
|
query: str,
|
|
*,
|
|
top_k: int = 5,
|
|
refresh_if_missing: bool = True,
|
|
) -> list[dict[str, Any]]:
|
|
self.search_calls.append((plan_id, query, top_k, refresh_if_missing))
|
|
if self.search_error:
|
|
raise self.search_error
|
|
return list(self.results)
|
|
|
|
|
|
def _make_workspace(context, *, vector_stub=None):
|
|
"""Build a coverage gap workspace with in-memory UoW."""
|
|
temp_dir = Path(tempfile.mkdtemp(prefix="cov-gap-"))
|
|
add_cleanup(context, lambda: shutil.rmtree(temp_dir, ignore_errors=True))
|
|
|
|
plan = Plan(
|
|
id=201,
|
|
project_id=1,
|
|
name="cov-plan",
|
|
prompt="coverage",
|
|
current=True,
|
|
)
|
|
project = Project(id=1, name="cov-project", path=temp_dir)
|
|
|
|
settings = Settings()
|
|
uow = _InMemoryUnitOfWork(plan)
|
|
|
|
service = ContextService(
|
|
settings=settings,
|
|
unit_of_work=uow,
|
|
vector_store_service=vector_stub,
|
|
)
|
|
|
|
context.temp_dir = temp_dir
|
|
context.cov_plan = plan
|
|
context.cov_project = project
|
|
context.cov_settings = settings
|
|
context.cov_uow = uow
|
|
context.cov_service = service
|
|
context.cov_context_files = []
|
|
|
|
# Stub list_files so the analysis methods get predictable file paths
|
|
def _list_files_stub(self, project=None):
|
|
return list(context.cov_context_files)
|
|
|
|
service.list_files = MethodType(_list_files_stub, service)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a coverage gap workspace")
|
|
def step_coverage_gap_workspace(context):
|
|
_make_workspace(context)
|
|
|
|
|
|
@given("an unreadable .agentsignore file in the workspace")
|
|
def step_unreadable_agentsignore(context):
|
|
ignore_file = context.temp_dir / ".agentsignore"
|
|
ignore_file.write_text("pattern_to_ignore\n")
|
|
# Remove read permission to trigger the except branch
|
|
ignore_file.chmod(0o000)
|
|
|
|
original_read_text = Path.read_text
|
|
|
|
def _read_text(path_self: Path, *args, **kwargs):
|
|
if path_self == ignore_file:
|
|
raise PermissionError("Simulated unreadable .agentsignore")
|
|
return original_read_text(path_self, *args, **kwargs)
|
|
|
|
patcher = patch.object(Path, "read_text", _read_text)
|
|
patcher.start()
|
|
add_cleanup(context, patcher.stop)
|
|
|
|
def restore():
|
|
with contextlib.suppress(Exception):
|
|
ignore_file.chmod(stat.S_IRUSR | stat.S_IWUSR)
|
|
|
|
add_cleanup(context, restore)
|
|
|
|
|
|
@given("a coverage gap workspace with LangSmith tracing enabled")
|
|
def step_workspace_langsmith_enabled(context):
|
|
_make_workspace(context)
|
|
context.cov_settings.langsmith_enabled = True
|
|
context.cov_settings.langsmith_api_key = "ls-fake-key-for-coverage"
|
|
context.cov_settings.langsmith_project = "cov-test-project"
|
|
# Also set environment variables that Settings._evaluate_langsmith_configuration needs
|
|
os.environ["LANGCHAIN_API_KEY"] = "ls-fake-key-for-coverage"
|
|
os.environ["LANGCHAIN_TRACING_V2"] = "true"
|
|
os.environ["LANGCHAIN_PROJECT"] = "cov-test-project"
|
|
|
|
def cleanup_env():
|
|
os.environ.pop("LANGCHAIN_API_KEY", None)
|
|
os.environ.pop("LANGCHAIN_TRACING_V2", None)
|
|
os.environ.pop("LANGCHAIN_PROJECT", None)
|
|
|
|
add_cleanup(context, cleanup_env)
|
|
|
|
|
|
@given("a coverage gap workspace with a failing vector store")
|
|
def step_workspace_failing_vector_store(context):
|
|
stub = _VectorStoreStub(enabled=True)
|
|
stub.refresh_error = ConfigurationError("Simulated config error for coverage")
|
|
_make_workspace(context, vector_stub=stub)
|
|
context.cov_vector_stub = stub
|
|
|
|
|
|
@given("a coverage gap workspace with no context files")
|
|
def step_workspace_no_context_files(context):
|
|
_make_workspace(context)
|
|
context.cov_context_files = []
|
|
|
|
|
|
@given('a coverage gap workspace with a context file "{filename}"')
|
|
def step_workspace_with_context_file(context, filename: str):
|
|
_make_workspace(context)
|
|
file_path = context.temp_dir / filename
|
|
file_path.write_text(
|
|
"import os\nimport sys\n\ndef hello():\n print('hello')\n",
|
|
encoding="utf-8",
|
|
)
|
|
context.cov_context_files = [str(file_path)]
|
|
|
|
|
|
@given("a coverage gap workspace with vector search enabled")
|
|
def step_workspace_vector_enabled(context):
|
|
stub = _VectorStoreStub(enabled=True)
|
|
_make_workspace(context, vector_stub=stub)
|
|
context.cov_vector_stub = stub
|
|
|
|
|
|
@given("a coverage gap workspace with vector search disabled")
|
|
def step_workspace_vector_disabled(context):
|
|
stub = _VectorStoreStub(enabled=False)
|
|
_make_workspace(context, vector_stub=stub)
|
|
context.cov_vector_stub = stub
|
|
|
|
|
|
@given("the coverage workspace project has no id")
|
|
def step_coverage_project_no_id(context):
|
|
context.cov_project.id = None
|
|
|
|
|
|
@given("the vector store stub has results for the coverage workspace")
|
|
def step_vector_stub_has_results(context):
|
|
context.cov_vector_stub.results = [
|
|
{"path": "doc.py", "score": 0.95, "snippet": "match"}
|
|
]
|
|
|
|
|
|
@given("the vector store search raises a ConfigurationError")
|
|
def step_vector_search_config_error(context):
|
|
context.cov_vector_stub.search_error = ConfigurationError(
|
|
"Simulated search config error"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I load agentsignore patterns for the workspace directory")
|
|
def step_load_agentsignore(context):
|
|
# Clear cache so the load is fresh
|
|
context.cov_service._agentsignore_cache.clear()
|
|
context.loaded_patterns = context.cov_service._load_agentsignore(context.temp_dir)
|
|
|
|
|
|
@when("I call _get_current_plan with a project that has no id")
|
|
def step_get_current_plan_no_id(context):
|
|
project_no_id = Project(id=None, name="no-id-project", path=context.temp_dir)
|
|
context.current_plan_result = context.cov_service._get_current_plan(project_no_id)
|
|
|
|
|
|
@when("I call _get_current_plan with the workspace project")
|
|
def step_get_current_plan_with_id(context):
|
|
context.current_plan_result = context.cov_service._get_current_plan(
|
|
context.cov_project
|
|
)
|
|
|
|
|
|
@when("I build a LangSmith config with tracing disabled")
|
|
def step_build_langsmith_disabled(context):
|
|
context.langsmith_result = context.cov_service._build_langsmith_config(
|
|
context.cov_project,
|
|
run_name="DisabledRun",
|
|
file_paths=["a.py"],
|
|
mode="sync",
|
|
)
|
|
|
|
|
|
@when("I build a LangSmith config with tracing enabled")
|
|
def step_build_langsmith_enabled(context):
|
|
context.langsmith_result = context.cov_service._build_langsmith_config(
|
|
context.cov_project,
|
|
run_name="EnabledRun",
|
|
file_paths=["a.py"],
|
|
mode="sync",
|
|
)
|
|
|
|
|
|
@when("I prepare an analysis config for the workspace project")
|
|
def step_prepare_analysis_config(context):
|
|
context.analysis_config = context.cov_service._prepare_analysis_config(
|
|
context.cov_project,
|
|
run_name="TestRun",
|
|
file_paths=["file.py"],
|
|
mode="sync",
|
|
)
|
|
|
|
|
|
@when("I trigger a vector index refresh for the workspace plan")
|
|
def step_trigger_vector_refresh(context):
|
|
context.refresh_exception = None
|
|
try:
|
|
context.cov_service._refresh_vector_index(context.cov_plan.id)
|
|
except Exception as exc:
|
|
context.refresh_exception = exc
|
|
|
|
|
|
@when("I request a context analysis agent from the coverage service")
|
|
def step_request_context_agent(context):
|
|
context.cov_agent = context.cov_service._get_context_agent()
|
|
|
|
|
|
@when("I run analyze_context on the workspace project")
|
|
def step_run_analyze_context(context):
|
|
context.analysis_result = context.cov_service.analyze_context(context.cov_project)
|
|
|
|
|
|
@when("I run analyze_context_async on the workspace project")
|
|
def step_run_analyze_context_async(context):
|
|
context.async_analysis_result = asyncio.run(
|
|
context.cov_service.analyze_context_async(context.cov_project)
|
|
)
|
|
|
|
|
|
@when("I stream analyze_context on the workspace project")
|
|
def step_stream_analyze_context(context):
|
|
context.streaming_events = list(
|
|
context.cov_service.analyze_context_streaming(context.cov_project)
|
|
)
|
|
|
|
|
|
@when("I async stream analyze_context on the workspace project")
|
|
def step_async_stream_analyze_context(context):
|
|
async def _collect():
|
|
events = []
|
|
async for event in context.cov_service.analyze_context_streaming_async(
|
|
context.cov_project
|
|
):
|
|
events.append(event)
|
|
return events
|
|
|
|
context.async_streaming_events = asyncio.run(_collect())
|
|
|
|
|
|
@when("I get the context summary from the coverage service")
|
|
def step_get_context_summary(context):
|
|
context.cov_summary = context.cov_service.get_context_summary(context.cov_project)
|
|
|
|
|
|
@when("I get the context dependencies from the coverage service")
|
|
def step_get_context_dependencies(context):
|
|
context.cov_dependencies = context.cov_service.get_context_dependencies(
|
|
context.cov_project
|
|
)
|
|
|
|
|
|
@when("I get relevant files with threshold {threshold:f} from the coverage service")
|
|
def step_get_relevant_files(context, threshold: float):
|
|
context.cov_relevant = context.cov_service.get_relevant_files(
|
|
context.cov_project, threshold=threshold
|
|
)
|
|
|
|
|
|
@when('I search the coverage context for "{query}"')
|
|
def step_search_coverage_context(context, query: str):
|
|
context.cov_search_results = context.cov_service.search_context(
|
|
context.cov_project, query
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the loaded patterns should be an empty list")
|
|
def step_patterns_empty(context):
|
|
assert context.loaded_patterns == [], (
|
|
f"Expected empty list, got {context.loaded_patterns}"
|
|
)
|
|
|
|
|
|
@then("the coverage _get_current_plan result should be None")
|
|
def step_cov_result_none(context):
|
|
assert context.current_plan_result is None, (
|
|
f"Expected None, got {context.current_plan_result}"
|
|
)
|
|
|
|
|
|
@then("the coverage _get_current_plan result should match the workspace plan")
|
|
def step_cov_result_current_plan(context):
|
|
assert context.current_plan_result is not None, "Expected a plan, got None"
|
|
assert context.current_plan_result.id == context.cov_plan.id, (
|
|
f"Expected plan id {context.cov_plan.id}, got {context.current_plan_result.id}"
|
|
)
|
|
|
|
|
|
@then("the LangSmith config result should be an empty dict")
|
|
def step_langsmith_empty(context):
|
|
assert context.langsmith_result == {}, (
|
|
f"Expected empty dict, got {context.langsmith_result}"
|
|
)
|
|
|
|
|
|
@then("the LangSmith config result should contain project metadata")
|
|
def step_langsmith_has_metadata(context):
|
|
result = context.langsmith_result
|
|
assert "metadata" in result, f"Missing metadata key: {result}"
|
|
metadata = result["metadata"]
|
|
assert metadata.get("project_id") == context.cov_project.id, metadata
|
|
assert metadata.get("service") == "ContextService", metadata
|
|
|
|
|
|
@then("the LangSmith config result should contain a project tag")
|
|
def step_langsmith_has_project_tag(context):
|
|
result = context.langsmith_result
|
|
tags = result.get("tags", [])
|
|
assert any(t.startswith("project:") for t in tags), (
|
|
f"No project tag found in {tags}"
|
|
)
|
|
|
|
|
|
@then("the analysis config should contain a configurable thread_id")
|
|
def step_config_has_thread_id(context):
|
|
configurable = context.analysis_config.get("configurable", {})
|
|
thread_id = configurable.get("thread_id", "")
|
|
assert thread_id.startswith("context-analysis-"), (
|
|
f"Unexpected thread_id: {thread_id}"
|
|
)
|
|
|
|
|
|
@then("the refresh should complete without raising an exception")
|
|
def step_refresh_no_exception(context):
|
|
assert context.refresh_exception is None, (
|
|
f"Unexpected exception: {context.refresh_exception}"
|
|
)
|
|
|
|
|
|
@then("the returned agent should be a ContextAnalysisAgent instance")
|
|
def step_agent_is_correct_type(context):
|
|
from cleveragents.agents.context_analysis import ContextAnalysisAgent
|
|
|
|
assert isinstance(context.cov_agent, ContextAnalysisAgent), (
|
|
f"Expected ContextAnalysisAgent, got {type(context.cov_agent)}"
|
|
)
|
|
|
|
|
|
@then("the analysis documents should be empty")
|
|
def step_analysis_docs_empty(context):
|
|
docs = context.analysis_result.get("documents", [])
|
|
assert len(docs) == 0, f"Expected 0 documents, got {len(docs)}"
|
|
|
|
|
|
@then("the analysis summary should say no files to analyze")
|
|
def step_analysis_summary_no_files(context):
|
|
summary = context.analysis_result.get("summary", "")
|
|
assert "no context files" in summary.lower(), f"Unexpected summary: {summary}"
|
|
|
|
|
|
@then("the analysis documents should not be empty")
|
|
def step_analysis_docs_not_empty(context):
|
|
docs = context.analysis_result.get("documents", [])
|
|
assert len(docs) > 0, "Expected at least 1 document"
|
|
|
|
|
|
@then("the async analysis documents should be empty")
|
|
def step_async_docs_empty(context):
|
|
docs = context.async_analysis_result.get("documents", [])
|
|
assert len(docs) == 0, f"Expected 0 async documents, got {len(docs)}"
|
|
|
|
|
|
@then("the async analysis summary should say no files to analyze")
|
|
def step_async_summary_no_files(context):
|
|
summary = context.async_analysis_result.get("summary", "")
|
|
assert "no context files" in summary.lower(), f"Unexpected async summary: {summary}"
|
|
|
|
|
|
@then("the async analysis documents should not be empty")
|
|
def step_async_docs_not_empty(context):
|
|
docs = context.async_analysis_result.get("documents", [])
|
|
assert len(docs) > 0, "Expected at least 1 async document"
|
|
|
|
|
|
@then("the only streaming event should indicate no files to analyze")
|
|
def step_streaming_no_files(context):
|
|
assert context.streaming_events == [
|
|
{"type": "complete", "summary": "No context files to analyze"}
|
|
], f"Unexpected streaming events: {context.streaming_events}"
|
|
|
|
|
|
@then("the streaming events should include workflow node results")
|
|
def step_streaming_has_nodes(context):
|
|
assert len(context.streaming_events) >= 1, (
|
|
f"Expected at least 1 event, got {len(context.streaming_events)}"
|
|
)
|
|
|
|
|
|
@then("the only async streaming event should indicate no files to analyze")
|
|
def step_async_streaming_no_files(context):
|
|
assert context.async_streaming_events == [
|
|
{"type": "complete", "summary": "No context files to analyze"}
|
|
], f"Unexpected async streaming events: {context.async_streaming_events}"
|
|
|
|
|
|
@then("the async streaming events should contain workflow results")
|
|
def step_async_streaming_has_events(context):
|
|
assert len(context.async_streaming_events) >= 1, (
|
|
f"Expected at least 1 async event, got {len(context.async_streaming_events)}"
|
|
)
|
|
|
|
|
|
@then("the summary should be a non-empty string from coverage service")
|
|
def step_summary_not_empty(context):
|
|
assert context.cov_summary and context.cov_summary.strip(), (
|
|
f"Summary is empty: {context.cov_summary!r}"
|
|
)
|
|
|
|
|
|
@then("the dependency result should be a dictionary")
|
|
def step_dependencies_is_dict(context):
|
|
assert isinstance(context.cov_dependencies, dict), (
|
|
f"Expected dict, got {type(context.cov_dependencies)}"
|
|
)
|
|
|
|
|
|
@then("the relevant files result should be a list of tuples")
|
|
def step_relevant_is_list(context):
|
|
assert isinstance(context.cov_relevant, list), (
|
|
f"Expected list, got {type(context.cov_relevant)}"
|
|
)
|
|
|
|
|
|
@then("the coverage search results should be empty")
|
|
def step_search_empty(context):
|
|
assert context.cov_search_results == [], (
|
|
f"Expected empty results, got {context.cov_search_results}"
|
|
)
|
|
|
|
|
|
@then("the coverage search results should not be empty")
|
|
def step_search_not_empty(context):
|
|
assert len(context.cov_search_results) > 0, "Expected non-empty search results"
|