hamza.khyari
1e606553d4
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 18s
CI / security (pull_request) Successful in 35s
CI / typecheck (pull_request) Successful in 38s
CI / unit_tests (pull_request) Successful in 2m32s
CI / integration_tests (pull_request) Successful in 3m18s
CI / docker (pull_request) Successful in 49s
CI / coverage (pull_request) Successful in 5m29s
CI / benchmark-regression (pull_request) Successful in 33m48s
Implement the UKOIndexer service that produces UKO triples from resources using pluggable domain-specific analyzers, wraps each triple with provenance metadata, and simultaneously indexes into text, vector, and graph backends. Key design decisions and components: - UKOIndexer orchestrates the full index lifecycle: add_resource, update_resource (remove-then-add), remove_resource, and maintenance triggers. Each operation fires lifecycle hooks (on_indexed, on_removed, on_error) so callers can observe progress. - Analyzer selection is pluggable via ContentAnalyzer protocol. The indexer accepts a registry mapping resource types to analyzers. PythonAnalyzer and MarkdownAnalyzer are provided as built-in implementations. - LocationContentReader protocol abstracts file I/O with a base_dir parameter for path-traversal prevention (post-resolve validation rejects paths escaping the base directory and non-regular files). - UKOTriple model includes a @model_validator ensuring at least one of object_uri or object_value is populated, preventing empty triples at construction time. - Triple removal uses scoped deletion via uko:sourceResource predicate to avoid shared-subject collision — only triples originating from the specific resource are removed, not all triples for a shared subject. - _resource_subjects.pop is deferred until after all backend removal operations succeed, preventing inconsistent state on partial failure. - analyzer.analyze() is wrapped in try/except so that analyzer errors produce an IndexResult with error details rather than propagating exceptions to callers. - All lifecycle hook calls are guarded via _fire_on_indexed, _fire_on_removed, and _fire_on_error helpers that catch and log hook exceptions without disrupting the indexing pipeline. - max_triples parameter (default 50,000) bounds analyzer output size to prevent runaway resource consumption. - ResourceFileWatcher monitors filesystem paths via watchdog and triggers re-indexing callbacks on file changes with configurable debouncing. Emits RESOURCE_MODIFIED domain events via EventBus when file changes are detected. Debounce timers coalesce rapid edits into a single callback invocation. Thread-safe design with daemon threads for clean shutdown. - SearchResult.__post_init__ validates score is in [0.0, 1.0], correctly rejecting NaN values. - Placeholder embedding uses [1.0] instead of [float(len(content))] to avoid leaking content size information. - isinstance check on graph_backend ensures GraphIndexBackend protocol compliance at runtime. - Test doubles extracted to features/mocks/uko_indexer_mocks.py for reuse across BDD steps and Robot helpers. Spec reference: Architecture > ACMS > Real-time Index Synchronization (specification.md lines ~43205-43300). ISSUES CLOSED: #578
293 lines
8.3 KiB
Python
293 lines
8.3 KiB
Python
"""Test doubles for UKO Indexer BDD and Robot tests.
|
|
|
|
Provides in-memory content readers, failing backends, tracking
|
|
lifecycle hooks, and event bus stubs used by
|
|
``features/steps/uko_indexer_steps.py``,
|
|
``robot/helper_uko_indexer.py``, and ``benchmarks/uko_indexer_bench.py``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
from cleveragents.domain.models.acms.index_backends import (
|
|
IndexedDocument,
|
|
SearchResult,
|
|
)
|
|
from cleveragents.domain.models.acms.provenance import IndexResult
|
|
from cleveragents.domain.models.core.resource import Resource
|
|
from cleveragents.infrastructure.events.models import DomainEvent
|
|
from cleveragents.infrastructure.events.types import EventType
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Content readers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class InMemoryContentReader:
|
|
"""Content reader that returns pre-configured content strings."""
|
|
|
|
def __init__(self) -> None:
|
|
self._content: dict[str, str] = {}
|
|
|
|
def set_content(self, resource_id: str, content: str) -> None:
|
|
self._content[resource_id] = content
|
|
|
|
def read_content(self, resource: Resource) -> str:
|
|
if resource.resource_id in self._content:
|
|
return self._content[resource.resource_id]
|
|
raise OSError(f"No content for {resource.resource_id}")
|
|
|
|
|
|
class FailingContentReader:
|
|
"""Content reader that always raises OSError."""
|
|
|
|
def read_content(self, resource: Resource) -> str:
|
|
raise OSError("Simulated read failure")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lifecycle hooks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TrackingLifecycleHook:
|
|
"""Lifecycle hook that records events for assertion."""
|
|
|
|
def __init__(self) -> None:
|
|
self.indexed_events: list[IndexResult] = []
|
|
self.removed_events: list[tuple[str, str]] = []
|
|
self.error_events: list[tuple[str, str]] = []
|
|
|
|
def on_indexed(self, result: IndexResult) -> None:
|
|
self.indexed_events.append(result)
|
|
|
|
def on_removed(self, resource_id: str, project: str) -> None:
|
|
self.removed_events.append((resource_id, project))
|
|
|
|
def on_error(self, resource_id: str, error: str) -> None:
|
|
self.error_events.append((resource_id, error))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing backends (test doubles)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FailingGraphBackend:
|
|
"""Graph backend that raises on add_triple."""
|
|
|
|
def add_triple(
|
|
self,
|
|
project: str,
|
|
subject: str,
|
|
predicate: str,
|
|
obj: str,
|
|
) -> None:
|
|
raise RuntimeError("Simulated graph failure")
|
|
|
|
def remove_triples(
|
|
self,
|
|
project: str,
|
|
subject: str | None,
|
|
predicate: str | None,
|
|
obj: str | None,
|
|
) -> None:
|
|
pass
|
|
|
|
def query(self, project: str, sparql: str) -> list[dict[str, str]]:
|
|
return []
|
|
|
|
def triple_count(self, project: str | None = None) -> int:
|
|
return 0
|
|
|
|
|
|
class FailingTextBackend:
|
|
"""Text backend that raises on index_document."""
|
|
|
|
def index_document(
|
|
self,
|
|
project: str,
|
|
doc_id: str,
|
|
content: str,
|
|
metadata: dict[str, str],
|
|
) -> IndexedDocument:
|
|
raise RuntimeError("Simulated text failure")
|
|
|
|
def search(
|
|
self,
|
|
project: str,
|
|
query: str,
|
|
limit: int = 20,
|
|
) -> list[SearchResult]:
|
|
return []
|
|
|
|
def remove_document(self, project: str, doc_id: str) -> None:
|
|
pass
|
|
|
|
def rebuild_index(self, project: str) -> None:
|
|
pass
|
|
|
|
@property
|
|
def document_count(self) -> int:
|
|
return 0
|
|
|
|
|
|
class FailingVectorBackend:
|
|
"""Vector backend that raises on index_embedding."""
|
|
|
|
def index_embedding(
|
|
self,
|
|
project: str,
|
|
doc_id: str,
|
|
embedding: list[float],
|
|
metadata: dict[str, str],
|
|
) -> None:
|
|
raise RuntimeError("Simulated vector failure")
|
|
|
|
def search_similar(
|
|
self,
|
|
project: str,
|
|
query_embedding: list[float],
|
|
limit: int = 20,
|
|
min_relevance: float = 0.0,
|
|
) -> list[SearchResult]:
|
|
return []
|
|
|
|
def remove_embedding(self, project: str, doc_id: str) -> None:
|
|
pass
|
|
|
|
@property
|
|
def embedding_count(self) -> int:
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Event bus stubs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TrackingEventBus:
|
|
"""Minimal EventBus stub that records emitted events."""
|
|
|
|
def __init__(self) -> None:
|
|
self.events: list[DomainEvent] = []
|
|
self._event = threading.Event()
|
|
|
|
def emit(self, event: DomainEvent) -> None:
|
|
self.events.append(event)
|
|
self._event.set()
|
|
|
|
def subscribe(
|
|
self,
|
|
event_type: EventType,
|
|
handler: Callable[..., Any],
|
|
) -> None:
|
|
_ = event_type, handler # Not needed for tests
|
|
|
|
def wait(self, timeout: float = 2.0) -> bool:
|
|
return self._event.wait(timeout=timeout)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backends that fail on removal (not on add)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class RemovalFailingGraphBackend:
|
|
"""Graph backend that works for add but fails on remove_triples."""
|
|
|
|
def __init__(self) -> None:
|
|
self._triples: list[tuple[str, str, str, str]] = []
|
|
|
|
def add_triple(self, project: str, subject: str, predicate: str, obj: str) -> None:
|
|
self._triples.append((project, subject, predicate, obj))
|
|
|
|
def remove_triples(
|
|
self, project: str, subject: str | None, predicate: str | None, obj: str | None
|
|
) -> None:
|
|
raise RuntimeError("Simulated graph removal failure")
|
|
|
|
def query(self, project: str, sparql: str) -> list[dict[str, str]]:
|
|
return []
|
|
|
|
def triple_count(self, project: str | None = None) -> int:
|
|
return len(self._triples)
|
|
|
|
|
|
class RemovalFailingTextBackend:
|
|
"""Text backend that works for add but fails on remove_document."""
|
|
|
|
def index_document(
|
|
self, project: str, doc_id: str, content: str, metadata: dict[str, str]
|
|
) -> IndexedDocument:
|
|
return IndexedDocument(project=project, doc_id=doc_id, char_count=len(content))
|
|
|
|
def search(self, project: str, query: str, limit: int = 20) -> list[SearchResult]:
|
|
return []
|
|
|
|
def remove_document(self, project: str, doc_id: str) -> None:
|
|
raise RuntimeError("Simulated text removal failure")
|
|
|
|
def rebuild_index(self, project: str) -> None:
|
|
pass
|
|
|
|
@property
|
|
def document_count(self) -> int:
|
|
return 0
|
|
|
|
|
|
class RemovalFailingVectorBackend:
|
|
"""Vector backend that works for add but fails on remove_embedding."""
|
|
|
|
def index_embedding(
|
|
self,
|
|
project: str,
|
|
doc_id: str,
|
|
embedding: list[float],
|
|
metadata: dict[str, str],
|
|
) -> None:
|
|
pass
|
|
|
|
def search_similar(
|
|
self,
|
|
project: str,
|
|
query_embedding: list[float],
|
|
limit: int = 20,
|
|
min_relevance: float = 0.0,
|
|
) -> list[SearchResult]:
|
|
return []
|
|
|
|
def remove_embedding(self, project: str, doc_id: str) -> None:
|
|
raise RuntimeError("Simulated vector removal failure")
|
|
|
|
@property
|
|
def embedding_count(self) -> int:
|
|
return 0
|
|
|
|
|
|
class SelectiveFailingGraphBackend:
|
|
"""Graph backend that fails only on specific predicates."""
|
|
|
|
def __init__(self, fail_on: set[str] | None = None) -> None:
|
|
self._triples: list[tuple[str, str, str, str]] = []
|
|
self._fail_on = fail_on or set()
|
|
|
|
def add_triple(self, project: str, subject: str, predicate: str, obj: str) -> None:
|
|
if predicate in self._fail_on:
|
|
raise RuntimeError(f"Simulated failure on {predicate}")
|
|
self._triples.append((project, subject, predicate, obj))
|
|
|
|
def remove_triples(
|
|
self, project: str, subject: str | None, predicate: str | None, obj: str | None
|
|
) -> None:
|
|
pass
|
|
|
|
def query(self, project: str, sparql: str) -> list[dict[str, str]]:
|
|
return []
|
|
|
|
def triple_count(self, project: str | None = None) -> int:
|
|
return len(self._triples)
|