diff --git a/features/steps/domain_models_steps.py b/features/steps/domain_models_steps.py index f959b040c..0d9e1535c 100644 --- a/features/steps/domain_models_steps.py +++ b/features/steps/domain_models_steps.py @@ -16,11 +16,10 @@ from cleveragents.domain.models.core import ( Context, ContextType, ContextUpdateResult, - CreditType, CreditsTransaction, CreditsTransactionType, + CreditType, Invite, - MaxContextCount, OperationType, Org, OrgRole, diff --git a/features/steps/vector_store_service_steps.py b/features/steps/vector_store_service_steps.py new file mode 100644 index 000000000..f42adf877 --- /dev/null +++ b/features/steps/vector_store_service_steps.py @@ -0,0 +1,429 @@ +"""Step definitions for vector store service coverage.""" + +from __future__ import annotations + +import shutil +import sys +import tempfile +import types +from collections.abc import Iterable +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from behave import given, then, when +from behave.runner import Context +from features.steps.service_steps import add_cleanup + +from cleveragents.application.services.vector_store_service import VectorStoreService +from cleveragents.config.settings import Settings +from cleveragents.core.exceptions import ConfigurationError +from cleveragents.domain.models.core.context import Context as ContextModel + + +class _StubContextRepository: + def __init__(self, backing_store: dict[int, list[ContextModel]]) -> None: + self._backing_store = backing_store + + def get_for_plan(self, plan_id: int) -> Iterable[ContextModel]: + return list(self._backing_store.get(plan_id, [])) + + +class _StubTransaction: + def __init__(self, unit: _StubUnitOfWork) -> None: + self._unit = unit + + def __enter__(self) -> _StubUnitOfWork: + return self._unit + + def __exit__(self, exc_type, exc, tb) -> bool: + return False + + +class _StubUnitOfWork: + def __init__(self) -> None: + self._contexts_map: dict[int, list[ContextModel]] = {} + self.contexts = _StubContextRepository(self._contexts_map) + + def transaction(self) -> _StubTransaction: + return _StubTransaction(self) + + def set_contexts(self, plan_id: int, contexts: list[ContextModel]) -> None: + self._contexts_map[plan_id] = contexts + + +class _StubDocument: + def __init__(self, path: str, content: str) -> None: + self.metadata = {"path": path} + self.page_content = content + + +class RecordingFAISS: + """Test double that records FAISS usage.""" + + last_from_texts_args: dict[str, Any] | None = None + last_built_instance: RecordingFAISS | None = None + load_local_should_raise: bool = False + load_local_calls: list[str] = [] + default_similarity_payload: list[tuple[_StubDocument, float]] = [] + + def __init__(self) -> None: + self.documents: list[str] = [] + self.metadatas: list[dict[str, Any]] = [] + self.saved_directory: str | None = None + self.similarity_payload: list[tuple[_StubDocument, float]] = [] + self.last_limit: int | None = None + self.last_query: str | None = None + + @classmethod + def reset(cls) -> None: + cls.last_from_texts_args = None + cls.last_built_instance = None + cls.load_local_should_raise = False + cls.load_local_calls = [] + cls.default_similarity_payload = [] + + @classmethod + def from_texts( + cls, + documents: list[str], + *, + embedding: Any, + metadatas: list[dict[str, Any]], + ) -> RecordingFAISS: + instance = cls() + instance.documents = list(documents) + instance.metadatas = list(metadatas) + instance.similarity_payload = list(cls.default_similarity_payload) + cls.last_from_texts_args = { + "documents": list(documents), + "metadatas": list(metadatas), + "embedding": embedding, + } + cls.last_built_instance = instance + return instance + + def save_local(self, directory: str) -> None: + self.saved_directory = directory + + def similarity_search_with_score( + self, query: str, k: int + ) -> list[tuple[_StubDocument, float]]: + self.last_query = query + self.last_limit = k + return self.similarity_payload[:k] + + @classmethod + def load_local( + cls, + directory: str, + embeddings: Any, + allow_dangerous_deserialization: bool = True, + ) -> RecordingFAISS: + cls.load_local_calls.append(directory) + if cls.load_local_should_raise: + raise ValueError("Failed to load index") + instance = cls() + instance.saved_directory = directory + cls.last_built_instance = instance + return instance + + +class StubOpenAIEmbeddings: + """Minimal stub to capture requested OpenAI embedding model.""" + + last_model: str | None = None + + def __init__(self, *, model: str | None = None, **_) -> None: + StubOpenAIEmbeddings.last_model = model + + +def _create_vector_service(context: Context, *, enabled: bool) -> None: + temp_dir = tempfile.mkdtemp(prefix="vector-store-service-") + add_cleanup(context, lambda: shutil.rmtree(temp_dir, ignore_errors=True)) + settings = Settings() + settings.vector_store_enabled = enabled + settings.vector_store_path = Path(temp_dir) / "vector_store" + settings.vector_embeddings_provider = "fake" + settings.vector_embeddings_dimension = 8 + unit = _StubUnitOfWork() + context.vector_store_service = VectorStoreService(settings, unit) + context.stub_unit_of_work = unit + context.vector_store_root = temp_dir + context.error = None + context.refresh_result = None + context.search_results = None + context.stub_openai_class = None + + +def _set_plan_contexts( + context: Context, + plan_id: int, + entries: list[tuple[str, str | None]], +) -> None: + base = Path(context.vector_store_root) + contexts = [ + ContextModel(plan_id=plan_id, path=str(base / rel_path), content=content) + for rel_path, content in entries + ] + context.stub_unit_of_work.set_contexts(plan_id, contexts) + + +@given("a vector store service with search enabled") +def step_vector_service_enabled(context: Context) -> None: + _create_vector_service(context, enabled=True) + + +@given("a vector store service with search disabled") +def step_vector_service_disabled(context: Context) -> None: + _create_vector_service(context, enabled=False) + + +@given("plan {plan_id:d} has no context documents") +def step_plan_no_contexts(context: Context, plan_id: int) -> None: + context.stub_unit_of_work.set_contexts(plan_id, []) + + +@given("plan {plan_id:d} has contexts with stored content and blanks") +def step_plan_mixed_contexts(context: Context, plan_id: int) -> None: + _set_plan_contexts( + context, + plan_id, + [ + (f"plan_{plan_id}_doc.md", "Context body"), + (f"plan_{plan_id}_empty.md", None), + ], + ) + + +@given("plan {plan_id:d} has contexts with stored content") +def step_plan_filled_contexts(context: Context, plan_id: int) -> None: + _set_plan_contexts( + context, + plan_id, + [ + (f"plan_{plan_id}_alpha.md", "Alpha document"), + (f"plan_{plan_id}_beta.md", "Beta document"), + ], + ) + + +@given("plan {plan_id:d} already has persisted FAISS files") +def step_plan_has_files(context: Context, plan_id: int) -> None: + plan_dir = context.vector_store_service._plan_store_dir(plan_id) + for filename in ("index.faiss", "index.pkl"): + (plan_dir / filename).write_bytes(b"stub") + + +@given("FAISS interactions are recorded") +def step_patch_faiss(context: Context) -> None: + RecordingFAISS.reset() + patcher = patch( + "cleveragents.application.services.vector_store_service.FAISS", + RecordingFAISS, + ) + patcher.start() + add_cleanup(context, patcher.stop) + context.faiss_class = RecordingFAISS + + +@given("future FAISS builds will return similarity hits") +def step_future_faiss_hits(context: Context) -> None: + assert hasattr(context, "faiss_class"), "FAISS interactions were not recorded" + payload = [(_StubDocument("doc.md", "C" * 600), 0.25)] + context.faiss_class.default_similarity_payload = payload + + def cleanup() -> None: + context.faiss_class.default_similarity_payload = [] + + add_cleanup(context, cleanup) + + +@given("plan {plan_id:d} cache contains similarity results") +def step_plan_cache_with_results(context: Context, plan_id: int) -> None: + store = context.faiss_class() + store.similarity_payload = [ + (_StubDocument("doc.md", "A" * 600), 0.25), + (_StubDocument("doc-two.md", "B" * 50), 0.9), + ] + context.vector_store_service._cache[plan_id] = store + context.cached_store = store + + +@given("loading the plan {plan_id:d} index will fail") +def step_fail_load(context: Context, plan_id: int) -> None: + context.faiss_class.load_local_should_raise = True + + +@given('the embeddings provider flag is "{provider}"') +def step_set_embeddings_provider(context: Context, provider: str) -> None: + settings = context.vector_store_service.settings.model_copy( + update={"vector_embeddings_provider": provider} + ) + context.vector_store_service.settings = settings + + +@given('the embeddings provider is "openai" using model "{model}"') +def step_set_openai_provider(context: Context, model: str) -> None: + settings = context.vector_store_service.settings.model_copy( + update={ + "vector_embeddings_provider": "openai", + "vector_embeddings_model": model, + } + ) + context.vector_store_service.settings = settings + + +@given("a stub OpenAI embeddings backend is available") +def step_stub_openai_backend(context: Context) -> None: + StubOpenAIEmbeddings.last_model = None + module = types.SimpleNamespace(OpenAIEmbeddings=StubOpenAIEmbeddings) + previous = sys.modules.get("langchain_openai") + sys.modules["langchain_openai"] = module + + def cleanup() -> None: + if previous is None: + sys.modules.pop("langchain_openai", None) + else: + sys.modules["langchain_openai"] = previous + + add_cleanup(context, cleanup) + context.stub_openai_class = StubOpenAIEmbeddings + + +@when("I attempt to refresh the vector store for plan {plan_id:d}") +@when("I refresh the vector store for plan {plan_id:d}") +def step_refresh_plan(context: Context, plan_id: int) -> None: + try: + context.refresh_result = context.vector_store_service.refresh_for_plan(plan_id) + context.error = None + except Exception as exc: + context.error = exc + context.refresh_result = None + + +@when('I search plan {plan_id:d} with the query "{query}" and limit {limit:d}') +def step_search_with_limit( + context: Context, plan_id: int, query: str, limit: int +) -> None: + try: + context.search_results = context.vector_store_service.search( + plan_id, + query, + top_k=limit, + refresh_if_missing=False, + ) + context.error = None + except Exception as exc: + context.error = exc + context.search_results = None + + +@when('I search plan {plan_id:d} with the query "{query}"') +def step_search_auto_refresh(context: Context, plan_id: int, query: str) -> None: + context.search_results = context.vector_store_service.search(plan_id, query) + + +@when('I search plan {plan_id:d} with the query ""') +def step_search_empty_query(context: Context, plan_id: int) -> None: + step_search_auto_refresh(context, plan_id, "") + + +@when('I search plan {plan_id:d} with the query "{query}" and refresh disabled') +def step_search_no_refresh(context: Context, plan_id: int, query: str) -> None: + context.search_results = context.vector_store_service.search( + plan_id, + query, + refresh_if_missing=False, + ) + + +@when("I invalidate the vector store without specifying a plan") +def step_invalidate_without_plan(context: Context) -> None: + context.vector_store_service.invalidate(None) + + +@then("the refresh result should be {expected:d} documents") +@then("the refresh result should be {expected:d} document") +def step_verify_refresh_result(context: Context, expected: int) -> None: + assert context.refresh_result == expected, context.refresh_result + + +@then("the plan {plan_id:d} cache should be empty") +def step_verify_cache_empty(context: Context, plan_id: int) -> None: + assert plan_id not in context.vector_store_service._cache + + +@then("the plan {plan_id:d} cache should still contain the cached store") +def step_verify_cache_still_present(context: Context, plan_id: int) -> None: + cached = context.vector_store_service._cache.get(plan_id) + assert cached is context.cached_store + + +@then("the plan {plan_id:d} persisted files should be removed") +def step_verify_files_removed(context: Context, plan_id: int) -> None: + plan_dir = context.vector_store_service._plan_store_dir(plan_id) + for filename in ("index.faiss", "index.pkl"): + assert not (plan_dir / filename).exists() + + +@then("FAISS should be built with {expected:d} cleaned document") +def step_verify_faiss_documents(context: Context, expected: int) -> None: + args = context.faiss_class.last_from_texts_args + assert args is not None + assert len(args["documents"]) == expected + assert len(args["metadatas"]) == expected + + +@then("the plan {plan_id:d} cache should hold the FAISS instance") +def step_verify_cache_holds_instance(context: Context, plan_id: int) -> None: + cached = context.vector_store_service._cache.get(plan_id) + assert cached is context.faiss_class.last_built_instance + + +@then("FAISS should be loaded for plan {plan_id:d}") +def step_verify_faiss_loaded(context: Context, plan_id: int) -> None: + assert hasattr(context, "faiss_class"), "FAISS interactions were not recorded" + expected_dir = str(context.vector_store_service._plan_store_dir(plan_id)) + assert context.faiss_class.load_local_calls, "No FAISS load_local calls recorded" + assert expected_dir in context.faiss_class.load_local_calls + + +@then( + 'the search results should include one formatted hit with path "{path}" and score {score:f}' +) +def step_verify_formatted_result(context: Context, path: str, score: float) -> None: + assert isinstance(context.search_results, list) + assert len(context.search_results) == 1 + hit = context.search_results[0] + assert hit["path"] == path + assert abs(hit["score"] - score) < 1e-9 + assert len(hit["snippet"]) == 500 + + +@then("the FAISS similarity search limit should be {expected:d}") +def step_verify_limit(context: Context, expected: int) -> None: + assert context.cached_store.last_limit == expected + + +@then("the search results should be empty") +def step_verify_empty_results(context: Context) -> None: + assert context.search_results == [] + + +@then("a configuration error should mention disabled vector store support") +def step_verify_disabled_error(context: Context) -> None: + assert isinstance(context.error, ConfigurationError) + assert "disabled" in str(context.error).lower() + + +@then("a configuration error should mention unsupported embeddings provider") +def step_verify_unsupported_provider(context: Context) -> None: + assert isinstance(context.error, ConfigurationError) + assert "unsupported" in str(context.error).lower() + + +@then('the last OpenAI embeddings model should be "{model}"') +def step_verify_openai_model(context: Context, model: str) -> None: + assert context.stub_openai_class is not None + assert context.stub_openai_class.last_model == model diff --git a/features/vector_store_service.feature b/features/vector_store_service.feature new file mode 100644 index 000000000..1263df14d --- /dev/null +++ b/features/vector_store_service.feature @@ -0,0 +1,103 @@ +Feature: Vector Store Service + As a developer + I want deterministic semantic search helpers + So that vector store functionality stays well-covered + + Scenario: Refresh rejects when vector store is disabled + Given a vector store service with search disabled + When I attempt to refresh the vector store for plan 1 + Then a configuration error should mention disabled vector store support + + Scenario: Search rejects when vector store is disabled + Given a vector store service with search disabled + When I search plan 1 with the query "Need answers" and limit 1 + Then a configuration error should mention disabled vector store support + + Scenario: Refresh removes cache and persisted files when contexts are empty + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 7 cache contains similarity results + And plan 7 already has persisted FAISS files + And plan 7 has no context documents + When I refresh the vector store for plan 7 + Then the refresh result should be 0 documents + And the plan 7 cache should be empty + And the plan 7 persisted files should be removed + + Scenario: Refresh indexes cleaned contexts into FAISS + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 3 has contexts with stored content and blanks + When I refresh the vector store for plan 3 + Then the refresh result should be 1 document + And FAISS should be built with 1 cleaned document + And the plan 3 cache should hold the FAISS instance + + Scenario: Refresh honors OpenAI provider configuration + Given a vector store service with search enabled + And a stub OpenAI embeddings backend is available + And the embeddings provider is "openai" using model "text-embedding-3-large" + And plan 4 has contexts with stored content + And FAISS interactions are recorded + When I refresh the vector store for plan 4 + Then the last OpenAI embeddings model should be "text-embedding-3-large" + And the refresh result should be 2 documents + + Scenario: Unsupported embeddings provider raises a configuration error + Given a vector store service with search enabled + And the embeddings provider flag is "replit" + And plan 6 has contexts with stored content + When I attempt to refresh the vector store for plan 6 + Then a configuration error should mention unsupported embeddings provider + + Scenario: Cached FAISS search formats similarity hits + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 5 cache contains similarity results + When I search plan 5 with the query "Need summary" and limit 1 + Then the FAISS similarity search limit should be 1 + And the search results should include one formatted hit with path "doc.md" and score 0.25 + + Scenario: Search loads persisted FAISS index when cache is empty + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 8 already has persisted FAISS files + When I search plan 8 with the query "Need summary" + Then FAISS should be loaded for plan 8 + And the plan 8 cache should hold the FAISS instance + + Scenario: Search returns empty when loading fails and refresh is disabled + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 9 already has persisted FAISS files + And loading the plan 9 index will fail + When I search plan 9 with the query "Need summary" and refresh disabled + Then the search results should be empty + + Scenario: Search aborts when refresh indexes nothing + Given a vector store service with search enabled + And plan 12 has no context documents + When I search plan 12 with the query "Need refresh" + Then the search results should be empty + + Scenario: Search rebuilds the vector index when nothing is cached + Given a vector store service with search enabled + And FAISS interactions are recorded + And future FAISS builds will return similarity hits + And plan 13 has contexts with stored content + When I search plan 13 with the query "Need refresh" + Then FAISS should be built with 2 cleaned document + And the plan 13 cache should hold the FAISS instance + And the search results should include one formatted hit with path "doc.md" and score 0.25 + + Scenario: Search trims blank queries + Given a vector store service with search enabled + When I search plan 10 with the query "" + Then the search results should be empty + + Scenario: Invalidation without a plan identifier is a no-op + Given a vector store service with search enabled + And FAISS interactions are recorded + And plan 11 cache contains similarity results + When I invalidate the vector store without specifying a plan + Then the plan 11 cache should still contain the cached store diff --git a/implementation_plan.md b/implementation_plan.md index c889c2354..56107346c 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -630,7 +630,11 @@ All 10 ADRs have been created in `docs/architecture/decisions/`: Notes: Capture DI graph decisions, concurrency insights, and compatibility concerns. **Phase 1 Catch-up Tasks (Must Do First):** -1. **Create ADR-011**: Document LangChain/LangGraph integration patterns including: + +- 2025-12-06: Re-scoped the req_res.py request/response model conversions to Stage 10.5 (API Model Conversion) because the schemas depend on the server endpoints planned for Phase 5+. Stage 2 now only tracks the CLI/domain models already converted; detailed per-model tasks remain under Stage 10.5 for execution alongside the server work. + + 1. **Create ADR-011**: Document LangChain/LangGraph integration patterns including: + - Graph design patterns for agent workflows - State management strategies using LangGraph - Provider abstraction via LangChain @@ -3812,60 +3816,7 @@ If you can do all of the above by end of Day 1, you're on track! - [X] Convert data_models.py core stubs (6 models): Project, Plan, Context, Operation, PlanBuild, PlanResult — already exist in `src/cleveragents/domain/models/core/` - [X] Convert data_models.py cloud/billing models (7 models): Org, User, OrgUser, Invite, OrgRole, CloudBillingFields, CreditsTransaction — implemented in `src/cleveragents/domain/models/core/org.py:1` with exports from `core/__init__.py:1` and `domain/models/__init__.py:1`, plus Behave coverage in `features/domain_models.feature:108` and Robot smoke tests in `robot/domain_models.robot:1`. - [X] Convert plan_model_settings.py (1 model): PlanSettings — implemented in `src/cleveragents/domain/models/plansettings/__init__.py` with proper type hints and Pydantic validation. - - [ ] Convert req_res.py (53 API models) - defer to Phase 5 when implementing server endpoints - - [ ] CreateEmailVerificationRequest - - [ ] CreateEmailVerificationResponse - - [ ] VerifyEmailPinRequest - - [ ] SignInRequest - - [ ] UiSignInToken - - [ ] CreateAccountRequest - - [ ] SessionResponse - - [ ] CreateOrgRequest - - [ ] ConvertTrialRequest - - [ ] CreateOrgResponse - - [ ] InviteRequest - - [ ] CreateProjectRequest - - [ ] CreateProjectResponse - - [ ] SetProjectPlanRequest - - [ ] RenameProjectRequest - - [ ] CreatePlanRequest - - [ ] CreatePlanResponse - - [ ] GetCurrentBranchByPlanIdRequest - - [ ] ListPlansRunningResponse - - [ ] TellPlanRequest - - [ ] BuildPlanRequest - - [ ] RespondMissingFileRequest - - [ ] LoadContextParams - - [ ] LoadContextResponse - - [ ] UpdateContextParams - - [ ] GetFileMapRequest - - [ ] GetFileMapResponse - - [ ] LoadCachedFileMapRequest - - [ ] LoadCachedFileMapResponse - - [ ] GetContextBodyRequest - - [ ] GetContextBodyResponse - - [ ] DeleteContextRequest - - [ ] DeleteContextResponse - - [ ] RejectFileRequest - - [ ] RejectFilesRequest - - [ ] RewindPlanRequest - - [ ] RewindPlanResponse - - [ ] LogResponse - - [ ] CreateBranchRequest - - [ ] UpdateSettingsRequest - - [ ] UpdateSettingsResponse - - [ ] UpdatePlanConfigRequest - - [ ] UpdateDefaultPlanConfigRequest - - [ ] GetPlanConfigResponse - - [ ] GetDefaultPlanConfigResponse - - [ ] ListUsersResponse - - [ ] ApplyPlanRequest - - [ ] RenamePlanRequest - - [ ] GetBuildStatusResponse - - [ ] CreditsLogRequest - - [ ] CreditsLogResponse - - [ ] CreditsSummaryResponse - - [ ] GetBalanceResponse + - [X] Convert req_res.py (53 API models) — Re-scoped to Stage 10.5 (API Model Conversion) because these schemas depend on the server endpoints planned for Phase 5+. See Stage 10.5 for the full per-model checklist; Stage 2 now tracks only the CLI/domain models already converted. - [X] Code: **Implement 5 Essential Commands First** - [X] `agents init` - Initialize new project (basic stub exists) - [X] `agents context-load ` - Add files/directories to context diff --git a/pyproject.toml b/pyproject.toml index 0a535b00a..e5165fbbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dependencies = [ # Additional AI/ML utilities "tiktoken>=0.7.0", # Token counting for OpenAI models "httpx>=0.27.0", # Better async HTTP client for API calls + "faiss-cpu>=1.7.4", # Vector store backend ] [project.optional-dependencies] diff --git a/robot/helper_domain_models.py b/robot/helper_domain_models.py index 8b327f3af..5ef02d972 100644 --- a/robot/helper_domain_models.py +++ b/robot/helper_domain_models.py @@ -2,18 +2,18 @@ from __future__ import annotations -from datetime import datetime, timezone -from decimal import Decimal import sys +from datetime import UTC, datetime +from decimal import Decimal +from cleveragents.domain.models.core.enums import ModelProvider from cleveragents.domain.models.core.org import ( CloudBillingFields, CreditsTransaction, CreditsTransactionType, + CreditType, Org, ) -from cleveragents.domain.models.core.enums import ModelProvider -from cleveragents.domain.models.core.org import CreditType def _org_test() -> None: @@ -25,7 +25,7 @@ def _org_test() -> None: auto_rebuy_to_balance=Decimal("15.0"), notify_threshold=Decimal("2.0"), max_threshold_per_month=Decimal("100.0"), - billing_cycle_started_at=datetime(2025, 1, 1, tzinfo=timezone.utc), + billing_cycle_started_at=datetime(2025, 1, 1, tzinfo=UTC), changed_billing_mode=False, trial_paid=False, stripe_subscription_id="sub-123", @@ -65,7 +65,7 @@ def _credits_test() -> None: debit_model_provider=ModelProvider.OPENAI, debit_model_name="gpt-4.1", debit_model_role="ModelRolePlanner", - created_at=datetime(2025, 1, 2, tzinfo=timezone.utc), + created_at=datetime(2025, 1, 2, tzinfo=UTC), ) assert tx.debit_model_provider == ModelProvider.OPENAI assert tx.amount == Decimal("10.0") diff --git a/src/cleveragents/application/services/plan_service.py b/src/cleveragents/application/services/plan_service.py index 0517b7f93..3d6c6608e 100644 --- a/src/cleveragents/application/services/plan_service.py +++ b/src/cleveragents/application/services/plan_service.py @@ -164,7 +164,7 @@ class PlanService: base_metadata |= metadata base_tags = ["service:plan"] - global_tags = getattr(self.settings, "langsmith_tags", None) or [] + global_tags = list(self.settings.langsmith_tags) if global_tags: base_tags.extend(global_tags) if project.id is not None: diff --git a/src/cleveragents/application/services/vector_store_service.py b/src/cleveragents/application/services/vector_store_service.py new file mode 100644 index 000000000..e7d9b2906 --- /dev/null +++ b/src/cleveragents/application/services/vector_store_service.py @@ -0,0 +1,179 @@ +"""Vector store management for semantic context search.""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from langchain_community.embeddings import FakeEmbeddings +from langchain_community.vectorstores.faiss import FAISS +from langchain_core.embeddings import Embeddings + +from cleveragents.config.settings import Settings +from cleveragents.core.exceptions import ConfigurationError +from cleveragents.domain.models.core import Context +from cleveragents.infrastructure.database.unit_of_work import UnitOfWork + + +class VectorStoreService: + """Manage creation and querying of LangChain vector stores.""" + + def __init__(self, settings: Settings, unit_of_work: UnitOfWork): + self.settings = settings + self.unit_of_work = unit_of_work + self._cache: dict[int, FAISS] = {} + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + def is_enabled(self) -> bool: + """Whether semantic vector search is enabled via settings.""" + + return bool(self.settings.vector_store_enabled) + + def invalidate(self, plan_id: int | None) -> None: + """Drop any cached vector store for the supplied plan.""" + + if plan_id is None: + return + self._cache.pop(plan_id, None) + + def refresh_for_plan(self, plan_id: int) -> int: + """Rebuild and persist the vector store for a plan. + + Returns the number of documents that were indexed. + """ + + self._ensure_enabled() + documents, metadata = self._prepare_documents(plan_id) + if not documents: + self.invalidate(plan_id) + self._remove_local_index(plan_id) + return 0 + + embeddings = self._create_embeddings() + vector_store = FAISS.from_texts( + documents, + embedding=embeddings, + metadatas=metadata, + ) + self._cache[plan_id] = vector_store + vector_store.save_local( + str(self._plan_store_dir(plan_id)), + ) + return len(documents) + + def search( + self, + plan_id: int, + query: str, + *, + top_k: int = 5, + refresh_if_missing: bool = True, + ) -> list[dict[str, Any]]: + """Run a similarity search against the plan's vector index.""" + + self._ensure_enabled() + query = query.strip() + if not query: + return [] + + store = self._cache.get(plan_id) or self._load_local_index(plan_id) + if store is None and refresh_if_missing: + if self.refresh_for_plan(plan_id) == 0: + return [] + store = self._cache.get(plan_id) or self._load_local_index(plan_id) + + if store is None: + return [] + + limit = max(1, top_k) + results = store.similarity_search_with_score(query, k=limit) + formatted: list[dict[str, Any]] = [] + for document, score in results: + formatted.append( + { + "path": document.metadata.get("path"), + "score": float(score), + "snippet": document.page_content[:500], + } + ) + return formatted + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + def _ensure_enabled(self) -> None: + if not self.is_enabled(): + raise ConfigurationError( + "Vector store support is disabled. Set " + "CLEVERAGENTS_VECTOR_STORE_ENABLED to true to enable semantic search." + ) + + def _prepare_documents( + self, plan_id: int + ) -> tuple[list[str], list[dict[str, Any]]]: + with self.unit_of_work.transaction() as ctx: + contexts: Iterable[Context] = ctx.contexts.get_for_plan(plan_id) + + documents: list[str] = [] + metadata: list[dict[str, Any]] = [] + for context in contexts: + if not context.content: + continue + documents.append(context.content) + metadata.append({"path": context.path, "plan_id": plan_id}) + return documents, metadata + + def _create_embeddings(self) -> Embeddings: + provider = (self.settings.vector_embeddings_provider or "fake").lower() + if provider == "fake": + return FakeEmbeddings(size=self.settings.vector_embeddings_dimension) + if provider in {"openai", "azure", "azureopenai"}: + try: + from langchain_openai import OpenAIEmbeddings + except ImportError as exc: # pragma: no cover - import guard + raise ConfigurationError( + "langchain-openai is required for OpenAI embeddings" + ) from exc + + model = self.settings.vector_embeddings_model or "text-embedding-3-small" + return OpenAIEmbeddings(model=model) + + raise ConfigurationError( + "Unsupported vector embeddings provider " + f"'{self.settings.vector_embeddings_provider}'" + ) + + def _plan_store_dir(self, plan_id: int) -> Path: + base = self.settings.resolve_vector_store_path() + plan_dir = base / f"plan_{plan_id}" + plan_dir.mkdir(parents=True, exist_ok=True) + return plan_dir + + def _load_local_index(self, plan_id: int) -> FAISS | None: + plan_dir = self._plan_store_dir(plan_id) + index_path = plan_dir / "index.faiss" + store_path = plan_dir / "index.pkl" + if not index_path.exists() or not store_path.exists(): + return None + embeddings = self._create_embeddings() + try: + store = FAISS.load_local( + str(plan_dir), + embeddings, + allow_dangerous_deserialization=True, + ) + except (ValueError, FileNotFoundError): + return None + self._cache[plan_id] = store + return store + + def _remove_local_index(self, plan_id: int) -> None: + plan_dir = self._plan_store_dir(plan_id) + index_path = plan_dir / "index.faiss" + store_path = plan_dir / "index.pkl" + for path in (index_path, store_path): + if path.exists(): + path.unlink(missing_ok=True) diff --git a/src/cleveragents/config/settings.py b/src/cleveragents/config/settings.py index 51d60c716..b5c4afecf 100644 --- a/src/cleveragents/config/settings.py +++ b/src/cleveragents/config/settings.py @@ -99,6 +99,32 @@ class Settings(BaseSettings): validation_alias=AliasChoices("CLEVERAGENTS_TEST_DATABASE_URL"), ) + # Vector store configuration + vector_store_enabled: bool = Field( + default=False, + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_STORE_ENABLED"), + ) + vector_store_backend: str = Field( + default="faiss", + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_STORE_BACKEND"), + ) + vector_store_path: Path = Field( + default_factory=lambda: Path(".cleveragents") / "vector_store", + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_STORE_PATH"), + ) + vector_embeddings_provider: str = Field( + default="fake", + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_EMBEDDINGS_PROVIDER"), + ) + vector_embeddings_model: str | None = Field( + default=None, + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_EMBEDDINGS_MODEL"), + ) + vector_embeddings_dimension: int = Field( + default=1536, + validation_alias=AliasChoices("CLEVERAGENTS_VECTOR_EMBEDDINGS_DIMENSION"), + ) + # LangSmith langsmith_enabled: bool = Field( default=False, @@ -203,6 +229,13 @@ class Settings(BaseSettings): return self.storage_path / storage_type return self.storage_path + def resolve_vector_store_path(self) -> Path: + """Return the absolute path for persisted vector store data.""" + base = self.vector_store_path + if not base.is_absolute(): + base = Path.cwd() / base + return base.resolve() + def get_database_url(self, *, test: bool = False) -> str: """Return the primary or test database URL.""" if test: @@ -289,11 +322,13 @@ class Settings(BaseSettings): if not self._get_langsmith_api_key(): errors.append( - "LangSmith API key is required (set CLEVERAGENTS_LANGSMITH_API_KEY or LANGCHAIN_API_KEY)." + "LangSmith API key is required " + "(set CLEVERAGENTS_LANGSMITH_API_KEY or LANGCHAIN_API_KEY)." ) if not self._get_langsmith_project(): errors.append( - "LangSmith project name is required (set CLEVERAGENTS_LANGSMITH_PROJECT or LANGCHAIN_PROJECT)." + "LangSmith project name is required " + "(set CLEVERAGENTS_LANGSMITH_PROJECT or LANGCHAIN_PROJECT)." ) return (len(errors) == 0, errors) diff --git a/src/cleveragents/domain/models/__init__.py b/src/cleveragents/domain/models/__init__.py index f2f94f282..93863bf25 100644 --- a/src/cleveragents/domain/models/__init__.py +++ b/src/cleveragents/domain/models/__init__.py @@ -17,9 +17,9 @@ from .core import ( ContextFile, ContextType, ContextUpdateResult, - CreditType, CreditsTransaction, CreditsTransactionType, + CreditType, Invite, MaxContextCount, Operation, @@ -52,6 +52,7 @@ __all__ = [ "Branch", "Change", "ChangeSet", + "CloudBillingFields", "Context", "ContextFile", "ContextType", @@ -59,12 +60,19 @@ __all__ = [ "ConvoMessage", "ConvoMessageFlags", "ConvoSummary", + "CreditType", + "CreditsTransaction", + "CreditsTransactionType", "CurrentPlanFiles", "CurrentPlanState", "CurrentStage", + "Invite", "MaxContextCount", "Operation", "OperationType", + "Org", + "OrgRole", + "OrgUser", "Plan", "PlanApply", "PlanBuild", @@ -81,13 +89,5 @@ __all__ = [ "Subtask", "SummaryForUpdateContextParams", "TellStage", - "CloudBillingFields", - "CreditType", - "CreditsTransaction", - "CreditsTransactionType", - "Invite", - "Org", - "OrgRole", - "OrgUser", "User", ] diff --git a/src/cleveragents/domain/models/core/__init__.py b/src/cleveragents/domain/models/core/__init__.py index ff68f41ce..16dbbd987 100644 --- a/src/cleveragents/domain/models/core/__init__.py +++ b/src/cleveragents/domain/models/core/__init__.py @@ -10,31 +10,39 @@ from .context import ( SummaryForUpdateContextParams, ) from .debug_attempt import DebugAttempt -from .plan import Plan, PlanBuild, PlanResult, PlanStatus from .org import ( CloudBillingFields, - CreditType, CreditsTransaction, CreditsTransactionType, + CreditType, Invite, Org, OrgRole, OrgUser, User, ) +from .plan import Plan, PlanBuild, PlanResult, PlanStatus from .project import Project, ProjectSettings, ProjectStats __all__ = [ "Change", "ChangeSet", + "CloudBillingFields", "Context", "ContextFile", "ContextType", "ContextUpdateResult", + "CreditType", + "CreditsTransaction", + "CreditsTransactionType", "DebugAttempt", + "Invite", "MaxContextCount", "Operation", "OperationType", + "Org", + "OrgRole", + "OrgUser", "Plan", "PlanBuild", "PlanResult", @@ -43,13 +51,5 @@ __all__ = [ "ProjectSettings", "ProjectStats", "SummaryForUpdateContextParams", - "CloudBillingFields", - "CreditType", - "CreditsTransaction", - "CreditsTransactionType", - "Invite", - "Org", - "OrgRole", - "OrgUser", "User", ] diff --git a/typings/langchain_community/embeddings/__init__.pyi b/typings/langchain_community/embeddings/__init__.pyi new file mode 100644 index 000000000..555f0ba36 --- /dev/null +++ b/typings/langchain_community/embeddings/__init__.pyi @@ -0,0 +1,14 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from langchain_core.embeddings import Embeddings + +class FakeEmbeddings(Embeddings): + """Deterministic embeddings generator used for testing.""" + + def __init__(self, *, size: int = ...) -> None: ... + def embed_documents(self, texts: Sequence[str]) -> list[list[float]]: ... + def embed_query(self, text: str) -> list[float]: ... + +__all__ = ["FakeEmbeddings"] diff --git a/typings/langchain_community/vectorstores/faiss.pyi b/typings/langchain_community/vectorstores/faiss.pyi new file mode 100644 index 000000000..9d88f4a19 --- /dev/null +++ b/typings/langchain_community/vectorstores/faiss.pyi @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any, Protocol + +from langchain_core.embeddings import Embeddings + +class _VectorDocument(Protocol): + metadata: dict[str, Any] + page_content: str + +class FAISS: + """Subset of the FAISS vector store methods used in CleverAgents.""" + + @classmethod + def from_texts( + cls, + texts: Sequence[str], + embedding: Embeddings, + metadatas: Sequence[dict[str, Any]] | None = ..., + ids: Sequence[str] | None = ..., + **kwargs: Any, + ) -> FAISS: ... + def save_local(self, folder_path: str) -> None: ... + @classmethod + def load_local( + cls, + folder_path: str, + embeddings: Embeddings, + *, + allow_dangerous_deserialization: bool = ..., + **kwargs: Any, + ) -> FAISS: ... + def similarity_search_with_score( + self, + query: str, + *, + k: int = ..., + filter: Any = ..., + fetch_k: int = ..., + **kwargs: Any, + ) -> list[tuple[_VectorDocument, float]]: ... + +__all__ = ["FAISS"] diff --git a/typings/langchain_core/embeddings/__init__.pyi b/typings/langchain_core/embeddings/__init__.pyi new file mode 100644 index 000000000..2086ddbdf --- /dev/null +++ b/typings/langchain_core/embeddings/__init__.pyi @@ -0,0 +1,9 @@ +from __future__ import annotations + +from collections.abc import Sequence + +class Embeddings: + """Minimal embeddings protocol used for typing.""" + + def embed_documents(self, texts: Sequence[str]) -> list[list[float]]: ... + def embed_query(self, text: str) -> list[float]: ... diff --git a/typings/langchain_openai/__init__.pyi b/typings/langchain_openai/__init__.pyi index 37b9bfcda..c7386f8a8 100644 --- a/typings/langchain_openai/__init__.pyi +++ b/typings/langchain_openai/__init__.pyi @@ -1,7 +1,9 @@ """Type stubs for langchain_openai package.""" +from collections.abc import Sequence from typing import Any +from langchain_core.embeddings import Embeddings as _Embeddings from langchain_core.language_models import BaseLanguageModel class ChatOpenAI(BaseLanguageModel): @@ -43,4 +45,18 @@ class AzureChatOpenAI(BaseLanguageModel): **kwargs: Any, ) -> None: ... -__all__ = ["ChatOpenAI", "AzureChatOpenAI"] +class OpenAIEmbeddings(_Embeddings): + """OpenAI embeddings wrapper used for vector stores.""" + + def __init__( + self, + *, + model: str = ..., + dimensions: int | None = None, + api_key: Any = None, + **kwargs: Any, + ) -> None: ... + def embed_documents(self, texts: Sequence[str]) -> list[list[float]]: ... + def embed_query(self, text: str) -> list[float]: ... + +__all__ = ["AzureChatOpenAI", "ChatOpenAI", "OpenAIEmbeddings"]