forked from HAL9000/cleveragents-core
1e606553d4
Implement the UKOIndexer service that produces UKO triples from resources using pluggable domain-specific analyzers, wraps each triple with provenance metadata, and simultaneously indexes into text, vector, and graph backends. Key design decisions and components: - UKOIndexer orchestrates the full index lifecycle: add_resource, update_resource (remove-then-add), remove_resource, and maintenance triggers. Each operation fires lifecycle hooks (on_indexed, on_removed, on_error) so callers can observe progress. - Analyzer selection is pluggable via ContentAnalyzer protocol. The indexer accepts a registry mapping resource types to analyzers. PythonAnalyzer and MarkdownAnalyzer are provided as built-in implementations. - LocationContentReader protocol abstracts file I/O with a base_dir parameter for path-traversal prevention (post-resolve validation rejects paths escaping the base directory and non-regular files). - UKOTriple model includes a @model_validator ensuring at least one of object_uri or object_value is populated, preventing empty triples at construction time. - Triple removal uses scoped deletion via uko:sourceResource predicate to avoid shared-subject collision — only triples originating from the specific resource are removed, not all triples for a shared subject. - _resource_subjects.pop is deferred until after all backend removal operations succeed, preventing inconsistent state on partial failure. - analyzer.analyze() is wrapped in try/except so that analyzer errors produce an IndexResult with error details rather than propagating exceptions to callers. - All lifecycle hook calls are guarded via _fire_on_indexed, _fire_on_removed, and _fire_on_error helpers that catch and log hook exceptions without disrupting the indexing pipeline. - max_triples parameter (default 50,000) bounds analyzer output size to prevent runaway resource consumption. - ResourceFileWatcher monitors filesystem paths via watchdog and triggers re-indexing callbacks on file changes with configurable debouncing. Emits RESOURCE_MODIFIED domain events via EventBus when file changes are detected. Debounce timers coalesce rapid edits into a single callback invocation. Thread-safe design with daemon threads for clean shutdown. - SearchResult.__post_init__ validates score is in [0.0, 1.0], correctly rejecting NaN values. - Placeholder embedding uses [1.0] instead of [float(len(content))] to avoid leaking content size information. - isinstance check on graph_backend ensures GraphIndexBackend protocol compliance at runtime. - Test doubles extracted to features/mocks/uko_indexer_mocks.py for reuse across BDD steps and Robot helpers. Spec reference: Architecture > ACMS > Real-time Index Synchronization (specification.md lines ~43205-43300). ISSUES CLOSED: #578
501 lines
17 KiB
Python
501 lines
17 KiB
Python
"""Analyzer lookup + core indexing + lifecycle + reindex steps.
|
|
|
|
All steps prefixed with ``idx`` to avoid AmbiguousStep collisions with
|
|
other feature files.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from behave import given, then, when # type: ignore[import-untyped]
|
|
|
|
from cleveragents.application.services.uko_indexer import UKOIndexer
|
|
from cleveragents.domain.models.acms.markdown_analyzer import MarkdownAnalyzer
|
|
from cleveragents.domain.models.acms.python_analyzer import PythonAnalyzer
|
|
from features.mocks.uko_indexer_mocks import (
|
|
FailingContentReader,
|
|
FailingGraphBackend,
|
|
FailingTextBackend,
|
|
FailingVectorBackend,
|
|
TrackingLifecycleHook,
|
|
)
|
|
from features.steps.uko_indexer_common import (
|
|
DEFAULT_PROJECT,
|
|
ULID_1,
|
|
_make_resource,
|
|
)
|
|
|
|
__all__: list[str] = []
|
|
|
|
# AnalyzerRegistry.get_for_resource steps
|
|
|
|
|
|
@given("idx a PythonAnalyzer is registered")
|
|
def step_register_python(context: Any) -> None:
|
|
context.idx_registry.register(PythonAnalyzer())
|
|
|
|
|
|
@given("idx a PythonAnalyzer is registered with priority {priority:d}")
|
|
def step_register_python_priority(context: Any, priority: int) -> None:
|
|
context.idx_registry.register(PythonAnalyzer(), priority=priority)
|
|
|
|
|
|
@given(
|
|
'idx an alternative Python analyzer "{name}" is registered with priority {priority:d}'
|
|
)
|
|
def step_register_alt_python(context: Any, name: str, priority: int) -> None:
|
|
"""Register a stub ``.py`` analyzer with custom *name* as domain."""
|
|
from cleveragents.domain.models.acms.analyzers import UKOTriple
|
|
|
|
class _Alt:
|
|
supported_extensions = frozenset({".py"})
|
|
domain = name
|
|
|
|
def analyze(self, content: str, resource_uri: str) -> list[UKOTriple]:
|
|
return []
|
|
|
|
context.idx_registry.register(_Alt(), priority=priority)
|
|
|
|
|
|
@given("idx a MarkdownAnalyzer is registered")
|
|
def step_register_markdown(context: Any) -> None:
|
|
context.idx_registry.register(MarkdownAnalyzer())
|
|
|
|
|
|
@when('idx I look up analyzer for a resource with location "{location}"')
|
|
def step_lookup_by_location(context: Any, location: str) -> None:
|
|
resource = _make_resource(ULID_1, location=location)
|
|
context.idx_found_analyzer = context.idx_registry.get_for_resource(resource)
|
|
|
|
|
|
@when("idx I look up analyzer for a resource with no location")
|
|
def step_lookup_no_location(context: Any) -> None:
|
|
resource = _make_resource(ULID_1, location=None)
|
|
context.idx_found_analyzer = context.idx_registry.get_for_resource(resource)
|
|
|
|
|
|
@then('idx the analyzer domain should be "{domain}"')
|
|
def step_check_analyzer_domain(context: Any, domain: str) -> None:
|
|
assert context.idx_found_analyzer is not None, "Expected an analyzer"
|
|
assert context.idx_found_analyzer.domain == domain
|
|
|
|
|
|
@then("idx no analyzer should be found")
|
|
def step_no_analyzer(context: Any) -> None:
|
|
assert context.idx_found_analyzer is None
|
|
|
|
|
|
# UKOIndexer setup steps
|
|
|
|
|
|
@given("idx a UKOIndexer with all backends")
|
|
def step_indexer_all(context: Any) -> None:
|
|
reader = context.idx_content_reader
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer without text backend")
|
|
def step_indexer_no_text(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=None,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer without vector backend")
|
|
def step_indexer_no_vector(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=None,
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with only graph backend")
|
|
def step_indexer_graph_only(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=None,
|
|
vector_backend=None,
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a failing content reader")
|
|
def step_indexer_failing_reader(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=FailingContentReader(),
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a tracking lifecycle hook")
|
|
def step_indexer_tracking_hook(context: Any) -> None:
|
|
context.idx_hook = TrackingLifecycleHook()
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=context.idx_content_reader,
|
|
lifecycle_hook=context.idx_hook,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a tracking lifecycle hook and failing reader")
|
|
def step_indexer_tracking_failing(context: Any) -> None:
|
|
context.idx_hook = TrackingLifecycleHook()
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=FailingContentReader(),
|
|
lifecycle_hook=context.idx_hook,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a failing graph backend")
|
|
def step_indexer_failing_graph(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=FailingGraphBackend(), # type: ignore[arg-type]
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a failing text backend")
|
|
def step_indexer_failing_text(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=FailingTextBackend(), # type: ignore[arg-type]
|
|
vector_backend=context.idx_vector_backend,
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
@given("idx a UKOIndexer with a failing vector backend")
|
|
def step_indexer_failing_vector(context: Any) -> None:
|
|
context.idx_indexer = UKOIndexer(
|
|
analyzer_registry=context.idx_registry,
|
|
graph_backend=context.idx_graph_backend,
|
|
text_backend=context.idx_text_backend,
|
|
vector_backend=FailingVectorBackend(), # type: ignore[arg-type]
|
|
content_reader=context.idx_content_reader,
|
|
)
|
|
|
|
|
|
# UKOIndexer action steps
|
|
|
|
|
|
@when('idx I index a Python resource "{rid}" with content "{content}"')
|
|
def step_index_python(context: Any, rid: str, content: str) -> None:
|
|
resource = _make_resource(rid, location="src/main.py")
|
|
# Unescape literal \n from Gherkin steps into real newlines
|
|
real_content = content.replace("\\n", "\n")
|
|
context.idx_content_reader.set_content(rid, real_content)
|
|
context.idx_result = context.idx_indexer.index_resource(
|
|
resource,
|
|
project=DEFAULT_PROJECT,
|
|
)
|
|
|
|
|
|
@when('idx I index a Markdown resource "{rid}" with content "{content}"')
|
|
def step_index_markdown(context: Any, rid: str, content: str) -> None:
|
|
resource = _make_resource(rid, location="README.md")
|
|
real_content = content.replace("\\n", "\n")
|
|
context.idx_content_reader.set_content(rid, real_content)
|
|
context.idx_result = context.idx_indexer.index_resource(
|
|
resource,
|
|
project=DEFAULT_PROJECT,
|
|
)
|
|
|
|
|
|
@when('idx I index a CSV resource "{rid}" with location "{location}"')
|
|
def step_index_csv(context: Any, rid: str, location: str) -> None:
|
|
resource = _make_resource(rid, location=location)
|
|
context.idx_content_reader.set_content(rid, "a,b,c")
|
|
context.idx_result = context.idx_indexer.index_resource(
|
|
resource,
|
|
project=DEFAULT_PROJECT,
|
|
)
|
|
|
|
|
|
@when('idx I remove resource "{rid}" from project "{project}"')
|
|
def step_remove_resource(context: Any, rid: str, project: str) -> None:
|
|
context.idx_indexer.remove_resource(rid, project=project)
|
|
|
|
|
|
@when('idx I reindex resource "{rid}" with content "{content}"')
|
|
def step_reindex_resource(context: Any, rid: str, content: str) -> None:
|
|
resource = _make_resource(rid, location="src/main.py")
|
|
real_content = content.replace("\\n", "\n")
|
|
context.idx_content_reader.set_content(rid, real_content)
|
|
context.idx_result = context.idx_indexer.reindex_resource(
|
|
resource,
|
|
project=DEFAULT_PROJECT,
|
|
)
|
|
|
|
|
|
@when(
|
|
'idx I index Python resource "{rid}" under project "{project}" content "{content}"'
|
|
)
|
|
def step_index_python_in_project(
|
|
context: Any, rid: str, project: str, content: str
|
|
) -> None:
|
|
resource = _make_resource(rid, location="src/main.py")
|
|
real_content = content.replace("\\n", "\n")
|
|
context.idx_content_reader.set_content(rid, real_content)
|
|
context.idx_result = context.idx_indexer.index_resource(
|
|
resource,
|
|
project=project,
|
|
)
|
|
|
|
|
|
@when('idx I reindex resource "{rid}" under project "{project}" content "{content}"')
|
|
def step_reindex_in_project(
|
|
context: Any,
|
|
rid: str,
|
|
project: str,
|
|
content: str,
|
|
) -> None:
|
|
resource = _make_resource(rid, location="src/main.py")
|
|
real_content = content.replace("\\n", "\n")
|
|
context.idx_content_reader.set_content(rid, real_content)
|
|
context.idx_result = context.idx_indexer.reindex_resource(
|
|
resource,
|
|
project=project,
|
|
)
|
|
|
|
|
|
# UKOIndexer assertion steps
|
|
|
|
|
|
@then("idx the index result should have triple_count greater than 0")
|
|
def step_check_triples_gt0(context: Any) -> None:
|
|
assert context.idx_result.triple_count > 0, (
|
|
f"Expected >0 triples, got {context.idx_result.triple_count}"
|
|
)
|
|
|
|
|
|
@then("idx the index result should have triple_count {count:d}")
|
|
def step_check_exact_triples(context: Any, count: int) -> None:
|
|
assert context.idx_result.triple_count == count
|
|
|
|
|
|
@then("idx the index result should have text_docs_indexed {count:d}")
|
|
def step_check_text_docs(context: Any, count: int) -> None:
|
|
assert context.idx_result.text_docs_indexed == count
|
|
|
|
|
|
@then("idx the index result should have embeddings_indexed {count:d}")
|
|
def step_check_embeddings(context: Any, count: int) -> None:
|
|
assert context.idx_result.embeddings_indexed == count
|
|
|
|
|
|
@then('idx the index result should have analyzer_domain "{domain}"')
|
|
def step_check_domain(context: Any, domain: str) -> None:
|
|
assert context.idx_result.analyzer_domain == domain
|
|
|
|
|
|
@then("idx the index result should have {count:d} errors")
|
|
def step_check_error_count(context: Any, count: int) -> None:
|
|
assert len(context.idx_result.errors) == count, (
|
|
f"Expected {count} errors, got {len(context.idx_result.errors)}: "
|
|
f"{context.idx_result.errors}"
|
|
)
|
|
|
|
|
|
@then('idx the index result error should mention "{word}"')
|
|
def step_check_error_message(context: Any, word: str) -> None:
|
|
assert any(word.lower() in e.lower() for e in context.idx_result.errors), (
|
|
f"No error mentioning '{word}' in {context.idx_result.errors}"
|
|
)
|
|
|
|
|
|
@then('idx the index result errors should mention "{word}"')
|
|
def step_check_errors_mention(context: Any, word: str) -> None:
|
|
assert any(word.lower() in e.lower() for e in context.idx_result.errors), (
|
|
f"No error mentioning '{word}' in {context.idx_result.errors}"
|
|
)
|
|
|
|
|
|
@then('idx the graph backend should have triples for project "{project}"')
|
|
def step_graph_has_triples(context: Any, project: str) -> None:
|
|
count = context.idx_graph_backend.triple_count(project)
|
|
assert count > 0, f"Expected triples, got {count}"
|
|
|
|
|
|
@then('idx the graph backend should have {count:d} triples for project "{project}"')
|
|
def step_graph_triple_count(context: Any, count: int, project: str) -> None:
|
|
actual = context.idx_graph_backend.triple_count(project)
|
|
assert actual == count, f"Expected {count}, got {actual}"
|
|
|
|
|
|
@then("idx the text backend should have {count:d} document")
|
|
def step_text_doc_count_singular(context: Any, count: int) -> None:
|
|
assert context.idx_text_backend.document_count == count
|
|
|
|
|
|
@then("idx the text backend should have {count:d} documents")
|
|
def step_text_doc_count_plural(context: Any, count: int) -> None:
|
|
assert context.idx_text_backend.document_count == count
|
|
|
|
|
|
@then("idx the vector backend should have {count:d} embedding")
|
|
def step_vector_count_singular(context: Any, count: int) -> None:
|
|
assert context.idx_vector_backend.embedding_count == count
|
|
|
|
|
|
@then("idx the vector backend should have {count:d} embeddings")
|
|
def step_vector_count_plural(context: Any, count: int) -> None:
|
|
assert context.idx_vector_backend.embedding_count == count
|
|
|
|
|
|
@then("idx the indexer should track {count:d} indexed resource")
|
|
def step_indexed_count_singular(context: Any, count: int) -> None:
|
|
assert context.idx_indexer.indexed_resource_count == count
|
|
|
|
|
|
@then("idx the indexer should track {count:d} indexed resources")
|
|
def step_indexed_count_plural(context: Any, count: int) -> None:
|
|
assert context.idx_indexer.indexed_resource_count == count
|
|
|
|
|
|
# Validation error steps
|
|
|
|
|
|
@then("idx indexing with empty project should raise ValueError")
|
|
def step_index_empty_project(context: Any) -> None:
|
|
resource = _make_resource(ULID_1)
|
|
try:
|
|
context.idx_indexer.index_resource(resource, project="")
|
|
assert False, "Expected ValueError" # noqa: B011
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
@then("idx removing with empty resource_id should raise ValueError")
|
|
def step_remove_empty_rid(context: Any) -> None:
|
|
try:
|
|
context.idx_indexer.remove_resource("", project=DEFAULT_PROJECT)
|
|
assert False, "Expected ValueError" # noqa: B011
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
@then("idx removing with empty project should raise ValueError")
|
|
def step_remove_empty_project(context: Any) -> None:
|
|
try:
|
|
context.idx_indexer.remove_resource(ULID_1, project="")
|
|
assert False, "Expected ValueError" # noqa: B011
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
# Lifecycle hook steps
|
|
|
|
|
|
@then("idx the lifecycle hook should have received {count:d} on_indexed event")
|
|
def step_hook_indexed_singular(context: Any, count: int) -> None:
|
|
assert len(context.idx_hook.indexed_events) == count
|
|
|
|
|
|
@then("idx the lifecycle hook should have received {count:d} on_indexed events")
|
|
def step_hook_indexed_plural(context: Any, count: int) -> None:
|
|
actual = len(context.idx_hook.indexed_events)
|
|
assert actual == count, f"Expected {count} on_indexed events, got {actual}"
|
|
|
|
|
|
@then("idx the lifecycle hook should have received {count:d} on_error events")
|
|
def step_hook_error(context: Any, count: int) -> None:
|
|
assert len(context.idx_hook.error_events) == count
|
|
|
|
|
|
@then("idx the lifecycle hook should have received {count:d} on_removed event")
|
|
def step_hook_removed(context: Any, count: int) -> None:
|
|
assert len(context.idx_hook.removed_events) == count
|
|
|
|
|
|
@then("idx the lifecycle hook should have received {count:d} on_removed events")
|
|
def step_hook_removed_plural(context: Any, count: int) -> None:
|
|
actual = len(context.idx_hook.removed_events)
|
|
assert actual == count, f"Expected {count} on_removed events, got {actual}"
|
|
|
|
|
|
# Constructor validation steps
|
|
|
|
|
|
@then("idx creating UKOIndexer with invalid registry should raise TypeError")
|
|
def step_invalid_registry(context: Any) -> None:
|
|
try:
|
|
UKOIndexer(
|
|
analyzer_registry="not a registry", # type: ignore[arg-type]
|
|
graph_backend=context.idx_graph_backend,
|
|
)
|
|
assert False, "Expected TypeError" # noqa: B011
|
|
except TypeError:
|
|
pass
|
|
|
|
|
|
@then("idx the indexer should have text backend")
|
|
def step_has_text(context: Any) -> None:
|
|
assert context.idx_indexer.has_text_backend is True
|
|
|
|
|
|
@then("idx the indexer should have vector backend")
|
|
def step_has_vector(context: Any) -> None:
|
|
assert context.idx_indexer.has_vector_backend is True
|
|
|
|
|
|
@then("idx the indexer should not have text backend")
|
|
def step_no_text(context: Any) -> None:
|
|
assert context.idx_indexer.has_text_backend is False
|
|
|
|
|
|
@then("idx the indexer should not have vector backend")
|
|
def step_no_vector(context: Any) -> None:
|
|
assert context.idx_indexer.has_vector_backend is False
|
|
|
|
|
|
# Reindex validation steps
|
|
|
|
|
|
@then("idx reindexing with empty project should raise ValueError")
|
|
def step_reindex_empty_project(context: Any) -> None:
|
|
resource = _make_resource(ULID_1)
|
|
try:
|
|
context.idx_indexer.reindex_resource(resource, project="")
|
|
assert False, "Expected ValueError" # noqa: B011
|
|
except ValueError:
|
|
pass
|
|
|
|
|
|
@then("idx reindexing with whitespace-only project should raise ValueError")
|
|
def step_reindex_whitespace_project(context: Any) -> None:
|
|
resource = _make_resource(ULID_1)
|
|
try:
|
|
context.idx_indexer.reindex_resource(resource, project=" ")
|
|
assert False, "Expected ValueError" # noqa: B011
|
|
except ValueError:
|
|
pass
|