Files
cleveragents-core/features/steps/uko_indexer_watcher_steps.py
T
hamza.khyari 1e606553d4
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 18s
CI / security (pull_request) Successful in 35s
CI / typecheck (pull_request) Successful in 38s
CI / unit_tests (pull_request) Successful in 2m32s
CI / integration_tests (pull_request) Successful in 3m18s
CI / docker (pull_request) Successful in 49s
CI / coverage (pull_request) Successful in 5m29s
CI / benchmark-regression (pull_request) Successful in 33m48s
feat(acms): implement Real-time Index Sync / UKOIndexer with pluggable analyzers
Implement the UKOIndexer service that produces UKO triples from resources
using pluggable domain-specific analyzers, wraps each triple with provenance
metadata, and simultaneously indexes into text, vector, and graph backends.

Key design decisions and components:

- UKOIndexer orchestrates the full index lifecycle: add_resource,
  update_resource (remove-then-add), remove_resource, and maintenance
  triggers. Each operation fires lifecycle hooks (on_indexed, on_removed,
  on_error) so callers can observe progress.

- Analyzer selection is pluggable via ContentAnalyzer protocol. The indexer
  accepts a registry mapping resource types to analyzers. PythonAnalyzer
  and MarkdownAnalyzer are provided as built-in implementations.

- LocationContentReader protocol abstracts file I/O with a base_dir
  parameter for path-traversal prevention (post-resolve validation rejects
  paths escaping the base directory and non-regular files).

- UKOTriple model includes a @model_validator ensuring at least one of
  object_uri or object_value is populated, preventing empty triples at
  construction time.

- Triple removal uses scoped deletion via uko:sourceResource predicate to
  avoid shared-subject collision — only triples originating from the
  specific resource are removed, not all triples for a shared subject.

- _resource_subjects.pop is deferred until after all backend removal
  operations succeed, preventing inconsistent state on partial failure.

- analyzer.analyze() is wrapped in try/except so that analyzer errors
  produce an IndexResult with error details rather than propagating
  exceptions to callers.

- All lifecycle hook calls are guarded via _fire_on_indexed,
  _fire_on_removed, and _fire_on_error helpers that catch and log hook
  exceptions without disrupting the indexing pipeline.

- max_triples parameter (default 50,000) bounds analyzer output size to
  prevent runaway resource consumption.

- ResourceFileWatcher monitors filesystem paths via watchdog and triggers
  re-indexing callbacks on file changes with configurable debouncing.
  Emits RESOURCE_MODIFIED domain events via EventBus when file changes
  are detected. Debounce timers coalesce rapid edits into a single
  callback invocation. Thread-safe design with daemon threads for clean
  shutdown.

- SearchResult.__post_init__ validates score is in [0.0, 1.0], correctly
  rejecting NaN values.

- Placeholder embedding uses [1.0] instead of [float(len(content))] to
  avoid leaking content size information.

- isinstance check on graph_backend ensures GraphIndexBackend protocol
  compliance at runtime.

- Test doubles extracted to features/mocks/uko_indexer_mocks.py for reuse
  across BDD steps and Robot helpers.

Spec reference: Architecture > ACMS > Real-time Index Synchronization
(specification.md lines ~43205-43300).

ISSUES CLOSED: #578
2026-03-11 00:40:07 +00:00

487 lines
16 KiB
Python

"""ResourceFileWatcher steps for UKO Indexer behave tests."""
from __future__ import annotations
import shutil
import tempfile
import threading
import time
from pathlib import Path
from typing import Any
from behave import given, then, when # type: ignore[import-untyped]
from watchdog.events import (
DirModifiedEvent,
FileCreatedEvent,
FileDeletedEvent,
FileModifiedEvent,
FileMovedEvent,
)
from cleveragents.application.services.resource_file_watcher import (
FileChangeType,
ResourceFileWatcher,
)
from cleveragents.infrastructure.events.types import EventType
from features.mocks.uko_indexer_mocks import TrackingEventBus
__all__: list[str] = []
def _cleanup_watcher(context: Any) -> None:
"""Stop the file watcher if running (behave cleanup hook)."""
watcher = getattr(context, "fw_watcher", None)
if watcher is not None and watcher.is_running:
watcher.stop()
tmpdir = getattr(context, "fw_tmpdir_obj", None)
if tmpdir is not None:
tmpdir.cleanup()
@given("idx a ResourceFileWatcher with default settings")
def step_fw_default(context: Any) -> None:
context.fw_watcher = ResourceFileWatcher()
context.add_cleanup(_cleanup_watcher, context)
@given("idx a ResourceFileWatcher with callback tracking and debounce {secs}")
def step_fw_tracking(context: Any, secs: str) -> None:
context.fw_changes = [] # list of (resource_id, project, change_type)
context.fw_callback_event = threading.Event()
def _on_change(rid: str, proj: str, ct: FileChangeType) -> None:
context.fw_changes.append((rid, proj, ct))
context.fw_callback_event.set()
context.fw_watcher = ResourceFileWatcher(
on_change=_on_change,
debounce_seconds=float(secs),
)
context.add_cleanup(_cleanup_watcher, context)
@given("idx a ResourceFileWatcher with EventBus tracking and debounce {secs}")
def step_fw_eventbus(context: Any, secs: str) -> None:
context.fw_event_bus = TrackingEventBus()
context.fw_watcher = ResourceFileWatcher(
event_bus=context.fw_event_bus,
debounce_seconds=float(secs),
)
context.add_cleanup(_cleanup_watcher, context)
@given("idx a ResourceFileWatcher with a failing callback and debounce {secs}")
def step_fw_failing_cb(context: Any, secs: str) -> None:
def _boom(rid: str, proj: str, ct: FileChangeType) -> None:
_ = rid, proj, ct
msg = "intentional test failure"
raise RuntimeError(msg)
context.fw_watcher = ResourceFileWatcher(
on_change=_boom,
debounce_seconds=float(secs),
)
context.add_cleanup(_cleanup_watcher, context)
@given('idx a temporary watched file "{name}" with content "{content}"')
def step_fw_temp_file(context: Any, name: str, content: str) -> None:
context.fw_tmpdir_obj = tempfile.TemporaryDirectory()
tmpdir = Path(context.fw_tmpdir_obj.name)
fpath = tmpdir / name
fpath.write_text(content)
context.fw_temp_file = fpath
@given("idx a temporary watched directory")
def step_fw_temp_dir(context: Any) -> None:
context.fw_tmpdir_obj = tempfile.TemporaryDirectory()
context.fw_temp_dir = Path(context.fw_tmpdir_obj.name)
@when("idx the file watcher is started")
@given("idx the file watcher is started")
def step_fw_start(context: Any) -> None:
context.fw_watcher.start()
@when("idx the file watcher is stopped")
def step_fw_stop(context: Any) -> None:
context.fw_watcher.stop()
@then("idx the file watcher should be running")
def step_fw_running(context: Any) -> None:
assert context.fw_watcher.is_running, "Watcher should be running"
@then("idx the file watcher should not be running")
def step_fw_not_running(context: Any) -> None:
assert not context.fw_watcher.is_running, "Watcher should not be running"
@when('idx the watcher registers the file for resource "{rid}" project "{proj}"')
@given('idx the watcher registers the file for resource "{rid}" project "{proj}"')
def step_fw_register(context: Any, rid: str, proj: str) -> None:
context.fw_watcher.watch(
context.fw_temp_file,
resource_id=rid,
project=proj,
)
@when("idx the watcher unregisters the watched file")
def step_fw_unregister(context: Any) -> None:
context.fw_watcher.unwatch(context.fw_temp_file)
@then("idx the watcher should have {n:d} watched paths")
def step_fw_count(context: Any, n: int) -> None:
actual = len(context.fw_watcher.watched_paths)
assert actual == n, f"Expected {n} watched paths, got {actual}"
@when("idx a FileModifiedEvent is simulated for the watched file")
def step_fw_sim_modified(context: Any) -> None:
event = FileModifiedEvent(str(context.fw_temp_file.resolve()))
context.fw_watcher._handle_fs_event(event)
@when("idx a FileDeletedEvent is simulated for the watched file")
def step_fw_sim_deleted(context: Any) -> None:
event = FileDeletedEvent(str(context.fw_temp_file.resolve()))
context.fw_watcher._handle_fs_event(event)
@when("idx 5 FileModifiedEvents are simulated rapidly for the watched file")
def step_fw_sim_rapid(context: Any) -> None:
for _ in range(5):
event = FileModifiedEvent(str(context.fw_temp_file.resolve()))
context.fw_watcher._handle_fs_event(event)
@when("idx we wait {secs} seconds for debounce")
def step_fw_wait(context: Any, secs: str) -> None:
time.sleep(float(secs))
@then("idx the on_change callback should fire within {secs:d} seconds")
def step_fw_cb_wait(context: Any, secs: int) -> None:
fired = context.fw_callback_event.wait(timeout=float(secs))
assert fired, f"Callback did not fire within {secs}s"
@then('idx the callback should receive resource "{rid}" project "{proj}" change "{ct}"')
def step_fw_cb_check(context: Any, rid: str, proj: str, ct: str) -> None:
assert len(context.fw_changes) > 0, "No callback invocations recorded"
last = context.fw_changes[-1]
assert last[0] == rid, f"resource_id: expected {rid}, got {last[0]}"
assert last[1] == proj, f"project: expected {proj}, got {last[1]}"
assert last[2] == ct, f"change_type: expected {ct}, got {last[2]}"
@then("idx the on_change callback should have fired exactly {n:d} time")
@then("idx the on_change callback should have fired exactly {n:d} times")
def step_fw_cb_count(context: Any, n: int) -> None:
actual = len(context.fw_changes)
assert actual == n, f"Expected {n} callback invocations, got {actual}"
@then(
"idx the EventBus should receive a RESOURCE_MODIFIED event within {secs:d} seconds"
)
def step_fw_eb_wait(context: Any, secs: int) -> None:
received = context.fw_event_bus.wait(timeout=float(secs))
assert received, f"No EventBus event within {secs}s"
assert len(context.fw_event_bus.events) > 0
assert context.fw_event_bus.events[-1].event_type == EventType.RESOURCE_MODIFIED
@then('idx the event details should contain resource_id "{rid}"')
def step_fw_eb_detail(context: Any, rid: str) -> None:
last = context.fw_event_bus.events[-1]
assert last.details.get("resource_id") == rid
@then('idx watching path "{path}" should raise ValueError containing "{substr}"')
def step_fw_watch_bad_path(context: Any, path: str, substr: str) -> None:
try:
context.fw_watcher.watch(
Path(path),
resource_id="r",
project="p",
)
assert False, "Expected ValueError" # noqa: B011
except ValueError as exc:
assert substr in str(exc), f"'{substr}' not in '{exc}'"
@then(
'idx watching the temporary directory should raise ValueError containing "{substr}"'
)
def step_fw_watch_dir(context: Any, substr: str) -> None:
try:
context.fw_watcher.watch(
context.fw_temp_dir,
resource_id="r",
project="p",
)
assert False, "Expected ValueError" # noqa: B011
except ValueError as exc:
assert substr in str(exc), f"'{substr}' not in '{exc}'"
@then("idx creating ResourceFileWatcher with debounce -1.0 should raise ValueError")
def step_fw_bad_debounce(context: Any) -> None:
try:
ResourceFileWatcher(debounce_seconds=-1.0)
assert False, "Expected ValueError" # noqa: B011
except ValueError:
pass
@when('idx a FileModifiedEvent is simulated for path "{path}"')
def step_fw_sim_unwatched(context: Any, path: str) -> None:
event = FileModifiedEvent(path)
context.fw_watcher._handle_fs_event(event)
@when("idx a directory event is simulated for the watched path")
def step_fw_sim_dir_event(context: Any) -> None:
event = DirModifiedEvent(str(context.fw_temp_file.resolve()))
context.fw_watcher._handle_fs_event(event)
@then("idx watching with empty resource_id should raise ValueError")
def step_fw_watch_empty_rid(context: Any) -> None:
try:
context.fw_watcher.watch(
context.fw_temp_file,
resource_id="",
project="p",
)
assert False, "Expected ValueError" # noqa: B011
except ValueError:
pass
@then("idx watching with empty project should raise ValueError")
def step_fw_watch_empty_project(context: Any) -> None:
try:
context.fw_watcher.watch(
context.fw_temp_file,
resource_id="r",
project="",
)
assert False, "Expected ValueError" # noqa: B011
except ValueError:
pass
@then("idx starting the watcher again should raise RuntimeError")
def step_fw_double_start(context: Any) -> None:
try:
context.fw_watcher.start()
assert False, "Expected RuntimeError" # noqa: B011
except RuntimeError:
pass
@when("idx I stop the file watcher")
@then("idx I stop the file watcher")
def step_fw_explicit_stop(context: Any) -> None:
context.fw_watcher.stop()
@then("idx the debounce_seconds should be {value}")
def step_fw_debounce_prop(context: Any, value: str) -> None:
assert context.fw_watcher.debounce_seconds == float(value), (
f"Expected {value}, got {context.fw_watcher.debounce_seconds}"
)
@when("idx a FileCreatedEvent is simulated for the watched file")
def step_fw_sim_created(context: Any) -> None:
event = FileCreatedEvent(str(context.fw_temp_file.resolve()))
context.fw_watcher._handle_fs_event(event)
@when("idx a FileMovedEvent is simulated for the watched file")
def step_fw_sim_moved(context: Any) -> None:
event = FileMovedEvent(
str(context.fw_temp_file.resolve()),
str(context.fw_temp_file.resolve()) + ".bak",
)
context.fw_watcher._handle_fs_event(event)
@given("idx a ResourceFileWatcher with auto_reindex disabled")
def step_fw_auto_reindex_disabled(context: Any) -> None:
context.fw_watcher = ResourceFileWatcher(auto_reindex=False)
context.add_cleanup(_cleanup_watcher, context)
@then("idx the auto_reindex property should be false")
def step_fw_auto_reindex_prop(context: Any) -> None:
assert context.fw_watcher.auto_reindex is False
@given("idx a ResourceFileWatcher with a directly-failing callback and debounce {secs}")
def step_fw_direct_fail_cb(context: Any, secs: str) -> None:
def _boom(rid: str, proj: str, ct: FileChangeType) -> None:
_ = rid, proj, ct
raise RuntimeError("intentional")
context.fw_watcher = ResourceFileWatcher(
on_change=_boom,
debounce_seconds=float(secs),
)
context.add_cleanup(_cleanup_watcher, context)
@when("idx _fire_change is called directly with failing callback")
def step_fw_fire_change_fail_cb(context: Any) -> None:
path = str(context.fw_temp_file.resolve())
context.fw_watcher._fire_change(
path,
"res-001",
"local/test",
FileChangeType.MODIFIED,
)
@given("idx a ResourceFileWatcher with a failing event bus and debounce {secs}")
def step_fw_failing_event_bus(context: Any, secs: str) -> None:
class _FailingBus:
def emit(self, event: object) -> None:
raise RuntimeError("bus boom")
def subscribe(self, event_type: object, handler: object) -> None:
pass
context.fw_watcher = ResourceFileWatcher(
event_bus=_FailingBus(), # type: ignore[arg-type]
debounce_seconds=float(secs),
)
context.add_cleanup(_cleanup_watcher, context)
@when("idx _fire_change is called directly with failing event bus")
def step_fw_fire_change_fail_bus(context: Any) -> None:
path = str(context.fw_temp_file.resolve())
context.fw_watcher._fire_change(
path,
"res-001",
"local/test",
FileChangeType.MODIFIED,
)
@when("idx _fire_change is called directly after stop")
def step_fw_fire_change_after_stop(context: Any) -> None:
path = str(context.fw_temp_file.resolve())
context.fw_watcher._fire_change(
path,
"res-001",
"local/test",
FileChangeType.MODIFIED,
)
@when("idx a _ResourceChangeHandler is used to route events")
def step_fw_handler_routes(context: Any) -> None:
from cleveragents.application.services.resource_file_watcher import (
_ResourceChangeHandler,
)
handler = _ResourceChangeHandler(context.fw_watcher)
path = str(context.fw_temp_file.resolve())
handler.on_modified(FileModifiedEvent(path))
handler.on_created(FileCreatedEvent(path))
handler.on_deleted(FileDeletedEvent(path))
handler.on_moved(FileMovedEvent(path, path + ".bak"))
@then("idx file watcher should fire both callback and event_bus on change")
def step_callback_and_event_bus_combo(context: Any) -> None:
td = tempfile.mkdtemp()
try:
fp = Path(td) / "combo.py"
fp.write_text("x=1\n")
chs: list[str] = []
ev = threading.Event()
uid = "01HQ8ZDRX50000000000000001"
bus = TrackingEventBus()
def cb(r: str, _p: str, _ct: FileChangeType) -> None:
chs.append(r)
ev.set()
w = ResourceFileWatcher(
on_change=cb,
event_bus=bus,
debounce_seconds=0.01,
)
w.start()
w.watch(fp, resource_id=uid, project="local/test")
w._handle_fs_event(FileModifiedEvent(str(fp.resolve())))
assert ev.wait(timeout=2.0) and chs[-1] == uid
assert bus.wait(timeout=2.0) and len(bus.events) >= 1
w.stop()
finally:
shutil.rmtree(td, ignore_errors=True)
@then("idx file watcher should schedule new dir on cross-directory move")
def step_fw_cross_dir_move(context: Any) -> None:
ev = threading.Event()
def cb(rid: str, _p: str, _ct: str) -> None:
ev.set()
td = tempfile.mkdtemp()
try:
sub = Path(td) / "sub"
sub.mkdir()
fp = Path(td) / "orig.py"
fp.write_text("x=1\n", encoding="utf-8")
dest = sub / "moved.py"
w = ResourceFileWatcher(on_change=cb, debounce_seconds=0.05)
w.watch(fp, resource_id="01HQ8ZDRX50000000000000077", project="local/t")
w.start()
w._handle_fs_event(FileMovedEvent(str(fp.resolve()), str(dest.resolve())))
assert ev.wait(timeout=2.0)
assert str(sub.resolve()) in w._dir_watches
w.stop()
finally:
shutil.rmtree(td, ignore_errors=True)
@then("idx file watcher should handle move then modify sequence")
def step_fw_move_then_modify(context: Any) -> None:
chs: list[str] = []
ev = threading.Event()
uid = "01HQ8ZDRX50000000000000099"
def cb(rid: str, _p: str, _ct: str) -> None:
chs.append(rid)
ev.set()
td = tempfile.mkdtemp()
try:
fp = Path(td) / "src.py"
fp.write_text("x=1\n", encoding="utf-8")
dp = str((Path(td) / "dst.py").resolve())
w = ResourceFileWatcher(on_change=cb, debounce_seconds=0.05)
w.watch(fp, resource_id=uid, project="local/test")
w.start()
w._handle_fs_event(FileMovedEvent(str(fp.resolve()), dp))
assert ev.wait(timeout=2.0), "Move cb"
ev.clear()
w._handle_fs_event(FileModifiedEvent(dp))
assert ev.wait(timeout=2.0), "Modify cb"
w.stop()
assert len(chs) >= 2 and chs[0] == uid and chs[1] == uid
finally:
shutil.rmtree(td, ignore_errors=True)