From 7d958121adf0f308470b806b025ba8dc97065669 Mon Sep 17 00:00:00 2001 From: Hamza Khyari Date: Fri, 6 Mar 2026 02:26:52 +0000 Subject: [PATCH] feat(context): add repo indexing service Add RepoIndexingService with incremental refresh, language detection, SHA-256 hashing, and policy enforcement. Includes domain models, DB persistence, DI wiring, Behave/Robot/ASV tests, and reference docs. ISSUES CLOSED: #195 --- CHANGELOG.md | 10 + .../versions/m7_001_repo_indexing_tables.py | 75 ++ benchmarks/context_indexing_bench.py | 130 ++++ docs/reference/context_indexing.md | 231 ++++++ features/repo_indexing.feature | 270 +++++++ features/steps/domain_analyzer_steps.py | 8 +- features/steps/repo_indexing_steps.py | 732 ++++++++++++++++++ robot/domain_analyzers.robot | 23 +- robot/helper_repo_indexing.py | 193 +++++ robot/repo_indexing.robot | 37 + src/cleveragents/application/container.py | 29 + .../application/services/__init__.py | 4 + .../services/repo_indexing_persistence.py | 276 +++++++ .../services/repo_indexing_service.py | 499 ++++++++++++ .../services/repo_indexing_utils.py | 382 +++++++++ .../domain/models/acms/_postgresql_helpers.py | 198 +++-- .../domain/models/acms/_sql_string_aware.py | 134 ++++ .../domain/models/acms/analyzers.py | 20 + .../models/acms/docker_compose_analyzer.py | 134 ++-- .../domain/models/acms/markdown_analyzer.py | 26 +- .../domain/models/acms/postgresql_analyzer.py | 26 +- .../domain/models/core/__init__.py | 12 + .../domain/models/core/repo_index.py | 268 +++++++ .../infrastructure/database/models.py | 86 ++ 24 files changed, 3644 insertions(+), 159 deletions(-) create mode 100644 alembic/versions/m7_001_repo_indexing_tables.py create mode 100644 benchmarks/context_indexing_bench.py create mode 100644 docs/reference/context_indexing.md create mode 100644 features/repo_indexing.feature create mode 100644 features/steps/repo_indexing_steps.py create mode 100644 robot/helper_repo_indexing.py create mode 100644 robot/repo_indexing.robot create mode 100644 src/cleveragents/application/services/repo_indexing_persistence.py create mode 100644 src/cleveragents/application/services/repo_indexing_service.py create mode 100644 src/cleveragents/application/services/repo_indexing_utils.py create mode 100644 src/cleveragents/domain/models/acms/_sql_string_aware.py create mode 100644 src/cleveragents/domain/models/core/repo_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index af4c890a7..9ef8c5553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +- Added `RepoIndexingService` for repository file indexing with incremental + refresh, extension-based language detection, SHA-256 content hashing, and + token estimation. Supports policy enforcement via include/exclude globs, + max file size, and max total size limits from project `ContextConfig`. + Persists index metadata and per-file records to SQLite via `RepoIndexModel` + and `IndexedFileModel`. Domain models (`IndexStatus`, `FileRecord`, + `IndexMetadata`, `RepoIndex`) are frozen Pydantic v2 with ULID IDs and UTC + datetimes. Wired into the DI container. Includes 28 Behave BDD scenarios, + 3 Robot Framework integration tests, ASV benchmarks (5 time + 2 track), and + reference documentation. (#195) - Fixed `agents project show` not finding a project immediately after creation. Extended the `session.commit()` fix from #589 to also cover `update()` and `delete()` in `NamespacedProjectRepository`, and updated the class docstring diff --git a/alembic/versions/m7_001_repo_indexing_tables.py b/alembic/versions/m7_001_repo_indexing_tables.py new file mode 100644 index 000000000..85f699d2f --- /dev/null +++ b/alembic/versions/m7_001_repo_indexing_tables.py @@ -0,0 +1,75 @@ +"""Add repo_indexes and indexed_files tables for repository indexing. + +Revision ID: m7_001_repo_indexing_tables +Revises: m6_003_async_jobs_table +Create Date: 2026-03-06 00:00:00 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "m7_001_repo_indexing_tables" +down_revision: str | Sequence[str] | None = "m6_003_async_jobs_table" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Create the repo_indexes and indexed_files tables.""" + op.create_table( + "repo_indexes", + sa.Column("index_id", sa.String(26), primary_key=True), + sa.Column("resource_id", sa.String(26), nullable=False, unique=True), + sa.Column("file_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("token_estimate", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "primary_language", + sa.String(50), + nullable=False, + server_default="unknown", + ), + sa.Column("status", sa.String(20), nullable=False, server_default="pending"), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("indexed_at", sa.String(40), nullable=False), + sa.Column("created_at", sa.String(40), nullable=False), + sa.CheckConstraint( + "status IN ('pending', 'indexing', 'ready', 'stale', 'error')", + name="ck_repo_indexes_status", + ), + ) + # resource_id already has unique=True which creates an implicit unique + # index — no redundant explicit index needed. + op.create_index("ix_repo_indexes_status", "repo_indexes", ["status"]) + + op.create_table( + "indexed_files", + sa.Column("index_id", sa.String(26), nullable=False, primary_key=True), + sa.Column("path", sa.String(1024), nullable=False, primary_key=True), + sa.Column("content_hash", sa.String(64), nullable=False), + sa.Column("token_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"), + sa.Column("language", sa.String(50), nullable=False, server_default="unknown"), + sa.Column("last_modified", sa.String(40), nullable=False), + sa.ForeignKeyConstraint( + ["index_id"], + ["repo_indexes.index_id"], + ondelete="CASCADE", + ), + ) + # index_id is the first column in the composite PK — SQLite already uses + # it for lookups; an additional single-column index is redundant. + # Note: FK ON DELETE CASCADE declared above is inactive without + # PRAGMA foreign_keys=ON; manual child-row deletion handles this. + op.create_index("ix_indexed_files_language", "indexed_files", ["language"]) + + +def downgrade() -> None: + """Drop the indexed_files and repo_indexes tables.""" + op.drop_index("ix_indexed_files_language", table_name="indexed_files") + op.drop_table("indexed_files") + op.drop_index("ix_repo_indexes_status", table_name="repo_indexes") + op.drop_table("repo_indexes") diff --git a/benchmarks/context_indexing_bench.py b/benchmarks/context_indexing_bench.py new file mode 100644 index 000000000..79856c652 --- /dev/null +++ b/benchmarks/context_indexing_bench.py @@ -0,0 +1,130 @@ +"""ASV benchmarks for repo indexing service (issue #195). + +Measures indexing throughput, incremental refresh overhead, and +language detection performance. +""" + +from __future__ import annotations + +import shutil +import sys +import tempfile +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402, F401 + +from sqlalchemy import create_engine # noqa: E402 +from sqlalchemy.orm import sessionmaker # noqa: E402 + +from cleveragents.application.services.repo_indexing_service import ( # noqa: E402 + RepoIndexingService, + detect_language, + estimate_tokens, +) +from cleveragents.infrastructure.database.models import Base # noqa: E402 + +_RID = "01KK0D8WNATFNEX2JMG5GKF6FP" + + +def _make_service() -> RepoIndexingService: + engine = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + return RepoIndexingService(session_factory=factory) + + +class TimeRepoIndexing: + """Benchmark repo indexing throughput.""" + + timeout = 120 + + def setup(self) -> None: + """Create a temporary project with 100 files.""" + self.svc = _make_service() + self.tmpdir = tempfile.mkdtemp() + for i in range(50): + (Path(self.tmpdir) / f"module_{i:03d}.py").write_text( + f"# Module {i}\ndef func_{i}():\n return {i}\n" * 10 + ) + for i in range(30): + (Path(self.tmpdir) / f"component_{i:03d}.ts").write_text( + f"export const comp{i} = () => {{}};\n" * 10 + ) + for i in range(20): + (Path(self.tmpdir) / f"doc_{i:03d}.md").write_text( + f"# Doc {i}\n\nContent for document {i}.\n" * 5 + ) + + def teardown(self) -> None: + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def time_full_index_100_files(self) -> None: + """Full index of 100 files.""" + self.svc.index_resource(_RID, self.tmpdir) + + def time_incremental_refresh_no_changes(self) -> None: + """Incremental refresh when no files changed (100 files).""" + # Pre-index to ensure an index exists, then measure refresh only. + self.svc.index_resource(_RID, self.tmpdir) + self.svc.refresh_index(_RID, self.tmpdir) + + def time_refresh_only_no_changes(self) -> None: + """Pure refresh measurement (index already exists). + + Note: ASV times the entire method body including the pre-index + call. For a purer measurement, use ``setup`` to pre-index and + only call ``refresh_index`` in the timed body. + """ + self.svc.index_resource(_RID, self.tmpdir) + self.svc.refresh_index(_RID, self.tmpdir) + + def time_language_detection_1000(self) -> None: + """Language detection for 1000 file paths.""" + paths = ( + [f"module_{i}.py" for i in range(250)] + + [f"component_{i}.ts" for i in range(250)] + + [f"doc_{i}.md" for i in range(250)] + + [f"data_{i}.json" for i in range(250)] + ) + for p in paths: + detect_language(p) + + def time_token_estimation_1000(self) -> None: + """Token estimation for 1000 sizes.""" + for size in range(1000): + estimate_tokens(size * 100) + + +class TrackRepoIndexing: + """Track metrics for repo indexing.""" + + timeout = 120 + + def setup(self) -> None: + self.svc = _make_service() + self.tmpdir = tempfile.mkdtemp() + for i in range(50): + (Path(self.tmpdir) / f"module_{i:03d}.py").write_text( + f"def func_{i}():\n return {i}\n" * 10 + ) + + def teardown(self) -> None: + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def track_file_count(self) -> int: + """Track number of files indexed.""" + idx = self.svc.index_resource(_RID, self.tmpdir) + return idx.metadata.file_count + + track_file_count.unit = "files" # type: ignore[attr-defined] + + def track_token_estimate(self) -> int: + """Track total token estimate.""" + idx = self.svc.index_resource(_RID, self.tmpdir) + return idx.metadata.token_estimate + + track_token_estimate.unit = "tokens" # type: ignore[attr-defined] diff --git a/docs/reference/context_indexing.md b/docs/reference/context_indexing.md new file mode 100644 index 000000000..42b9d4d98 --- /dev/null +++ b/docs/reference/context_indexing.md @@ -0,0 +1,231 @@ +# Repository Indexing Service + +The Repository Indexing Service scans linked repository resources, building a +persistent file-level index with language detection, content hashing, and token +estimation. ACMS uses this index for efficient context assembly on projects +with 10K+ files. + +## Architecture + +``` +resource_id + root_path + | + v + ┌─── RepoIndexingService ───┐ + │ walk filesystem │ + │ apply include/exclude globs│ + │ enforce max_file_size │ + │ enforce max_total_size │ + │ SHA-256 content hashing │ + │ extension language detect │ + │ estimate token counts │ + └──────────┬─────────────────┘ + v + ┌─── SQLite persistence ─────┐ + │ repo_indexes table │ + │ indexed_files table │ + └─────────────────────────────┘ +``` + +## Domain Models + +All models are frozen Pydantic v2 with ULID identifiers and UTC datetimes. + +### IndexStatus + +Enum representing the state of an index: + +| Value | Description | +|-------|-------------| +| `pending` | Index creation requested but not yet started | +| `indexing` | Filesystem walk in progress | +| `ready` | Index complete and available for queries | +| `error` | Indexing failed; see `error_message` | +| `stale` | Source files changed since last index | + +### FileRecord + +Per-file metadata stored during indexing: + +| Field | Type | Description | +|-------|------|-------------| +| `path` | `str` | Relative path from the repository root | +| `content_hash` | `str` | SHA-256 hex digest of file contents | +| `token_count` | `int` | Estimated token count (`size_bytes // 4`) | +| `language` | `str` | Detected programming language | +| `size_bytes` | `int` | File size in bytes | +| `last_modified` | `datetime` | File modification timestamp (UTC) | + +### IndexMetadata + +Summary record for a repository index: + +| Field | Type | Description | +|-------|------|-------------| +| `index_id` | `str` | ULID identifier for this index snapshot | +| `resource_id` | `str` | ULID of the linked resource | +| `indexed_at` | `datetime` | When indexing completed (UTC) | +| `file_count` | `int` | Total files in the index | +| `token_estimate` | `int` | Sum of all file token counts | +| `primary_language` | `str` | Most common language by token count (weighted) | +| `status` | `IndexStatus` | Current index state | +| `error_message` | `str | None` | Error details when `status == error` | + +### RepoIndex + +Composite object returned by index and refresh operations: + +| Field | Type | Description | +|-------|------|-------------| +| `metadata` | `IndexMetadata` | Index summary | +| `files` | `tuple[FileRecord, ...]` | All indexed file records | + +## Service API + +### `RepoIndexingService(session_factory)` + +Constructor. Accepts a SQLAlchemy session factory (injected via DI container). + +### `index_resource(resource_id, root_path, *, include_globs, exclude_globs, max_file_size, max_total_size) -> RepoIndex` + +Full index of a filesystem tree. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `resource_id` | `str` | required | ULID of the resource | +| `root_path` | `str \| Path` | required | Absolute path to the repository root | +| `include_globs` | `tuple[str, ...]` | `()` | Only index files matching these globs (empty = all) | +| `exclude_globs` | `tuple[str, ...]` | `()` | Skip files matching these globs | +| `max_file_size` | `int \| None` | `None` | Skip files larger than this (bytes); `None` = no limit | +| `max_total_size` | `int \| None` | `None` | Stop indexing when cumulative size exceeds this; `None` = no limit | + +Raises `ValueError` if `resource_id` is empty. Raises `FileNotFoundError` if +`root_path` does not exist. + +### `refresh_index(resource_id, root_path, **kwargs) -> RepoIndex` + +Incremental refresh. Compares content hashes against the existing index and +only re-processes changed files. Falls back to a full index if no prior index +exists. Accepts the same keyword arguments as `index_resource`. + +### `get_index(resource_id) -> RepoIndex | None` + +Retrieve a full index (metadata + file records) from the database. Returns +`None` if no index exists. + +### `get_index_status(resource_id) -> IndexMetadata | None` + +Lightweight query returning only the metadata (no file records). Used by +`agents project show` for status display. + +### `remove_index(resource_id) -> bool` + +Delete all index data for a resource. Returns `True` if records were deleted, +`False` if no index existed. + +### `cleanup_stale_indexing() -> int` + +Remove orphan `INDEXING` rows left by crashed processes. Should be called at +application startup. Returns the number of stale rows removed. + +## Language Detection + +Extension-based detection via `detect_language(path)`. Supported mappings: + +| Extensions | Language | +|-----------|----------| +| `.py`, `.pyi`, `.pyx` | python | +| `.ts`, `.tsx` | typescript | +| `.js`, `.jsx`, `.mjs`, `.cjs` | javascript | +| `.rs` | rust | +| `.java` | java | +| `.kt`, `.kts` | kotlin | +| `.go` | go | +| `.c`, `.h` | c | +| `.cpp`, `.cc`, `.cxx`, `.hpp` | cpp | +| `.cs` | csharp | +| `.rb` | ruby | +| `.php` | php | +| `.swift` | swift | +| `.scala` | scala | +| `.r` | r | +| `.md`, `.mdx` | markdown | +| `.rst` | restructuredtext | +| `.json` | json | +| `.yaml`, `.yml` | yaml | +| `.toml` | toml | +| `.xml` | xml | +| `.html`, `.htm` | html | +| `.css`, `.scss` | css | +| `.sql` | sql | +| `.sh`, `.bash`, `.zsh` | shell | +| `.ps1` | powershell | +| `.dockerfile` | dockerfile | +| `.tf` | terraform | +| `.lua` | lua | +| `.zig` | zig | +| `.nim` | nim | +| `.ex`, `.exs` | elixir | +| `.erl` | erlang | +| `.hs` | haskell | +| `.ml`, `.mli` | ocaml | +| `.clj` | clojure | +| `.dart` | dart | +| `.v` | v | +| `.jl` | julia | +| `Makefile`, `makefile`, `GNUmakefile` | makefile | +| `Dockerfile`, `Dockerfile.*` | dockerfile | + +Files with unrecognized extensions return `"unknown"`. + +## Configuration + +Indexing behaviour is controlled via the project's `ContextConfig` and +`ContextView`: + +| Config Key | Model Field | Description | +|-----------|-------------|-------------| +| `context.include_patterns` | `ContextConfig.include_patterns` | Include globs | +| `context.ignore_patterns` | `ContextConfig.ignore_patterns` | Exclude globs | +| `context.max_file_size` | `ContextConfig.max_file_size` | Per-file size limit | +| `context.max_total_size` | `ContextConfig.max_total_size` | Total index size cap | +| `context.indexing_strategy` | `ContextConfig.indexing_strategy` | `full_text` or `semantic` | +| `context.auto_refresh` | `ContextConfig.auto_refresh` | Auto-refresh on access | + +## Database Schema + +Two tables are added: + +### `repo_indexes` + +| Column | Type | Constraints | +|--------|------|-------------| +| `index_id` | `String(26)` | PK | +| `resource_id` | `String(26)` | NOT NULL, UNIQUE, INDEXED | +| `indexed_at` | `String(40)` | NOT NULL (ISO-8601 UTC) | +| `file_count` | `Integer` | NOT NULL, DEFAULT 0 | +| `token_estimate` | `Integer` | NOT NULL, DEFAULT 0 | +| `primary_language` | `String(50)` | NOT NULL, DEFAULT "unknown" | +| `status` | `String(20)` | NOT NULL, DEFAULT "pending", CHECK IN (`pending`, `indexing`, `ready`, `stale`, `error`) | +| `error_message` | `Text` | NULLABLE | +| `created_at` | `String(40)` | NOT NULL (ISO-8601 UTC) | + +### `indexed_files` + +| Column | Type | Constraints | +|--------|------|-------------| +| `index_id` | `String(26)` | PK (composite), FK -> `repo_indexes.index_id` ON DELETE CASCADE, INDEXED | +| `path` | `String(1024)` | PK (composite) | +| `content_hash` | `String(64)` | NOT NULL | +| `token_count` | `Integer` | NOT NULL, DEFAULT 0 | +| `size_bytes` | `Integer` | NOT NULL, DEFAULT 0 | +| `language` | `String(50)` | NOT NULL, DEFAULT "unknown" | +| `last_modified` | `String(40)` | NOT NULL | + +## Spec References + +- Lines 727-840: `agents info` repo indexing display +- Lines 2829-2916: `project link-resource` triggers indexing +- Lines 3322-3395: `project show` displays index status +- Lines 19719-19727: Project data model index fields +- Lines 28649-28664: `context.*` configuration keys diff --git a/features/repo_indexing.feature b/features/repo_indexing.feature new file mode 100644 index 000000000..d2e91eef6 --- /dev/null +++ b/features/repo_indexing.feature @@ -0,0 +1,270 @@ +# Repository indexing service tests targeting issue #195. +@feature195 +Feature: Repository indexing with incremental refresh and language detection + As a CleverAgents user with a large project + I want the system to index my repository files with language detection + So that context assembly can efficiently retrieve relevant code fragments + + # -- Full index ---------------------------------------------------------- + + @feature195 + Scenario: Full index of a repository with mixed file types + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index on the sample directory + Then the repo-index result status should be "ready" + And the repo-index result file count should be 6 + And the repo-index primary language should be "python" + And the repo-index token estimate should be positive + + @feature195 + Scenario: Indexed files have correct language detection + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index on the sample directory + Then the repo-index file "main.py" should have language "python" + And the repo-index file "index.ts" should have language "typescript" + And the repo-index file "README.md" should have language "markdown" + And the repo-index file "config.json" should have language "json" + And the repo-index file "Makefile" should have language "makefile" + + @feature195 + Scenario: Index is persisted and can be retrieved + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index on the sample directory + And I query repo-index for the indexed resource + Then the repo-index retrieved index should match the original + And the repo-index retrieved file hashes should match the original + + # -- Incremental refresh ------------------------------------------------- + + @feature195 + Scenario: Incremental refresh detects changed files + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I modify repo-index file "main.py" with new content + And I run repo-index incremental refresh + Then the repo-index file "main.py" content hash should differ from original + And the repo-index file "index.ts" content hash should match original + + @feature195 + Scenario: Refresh on non-existent index falls back to full index + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index incremental refresh + Then the repo-index result status should be "ready" + And the repo-index result file count should be 6 + + # -- Policy enforcement -------------------------------------------------- + + @feature195 + Scenario: Max file size excludes large files + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files and a 2MB file + When I run repo-index full index with max file size 1000000 bytes + Then the repo-index result should not include file "big_file.dat" + And the repo-index result file count should be 6 + + @feature195 + Scenario: Include globs filter to matching files only + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index with include globs "*.py" + Then the repo-index result file count should be 2 + And the repo-index primary language should be "python" + + @feature195 + Scenario: Exclude globs remove matching files + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index with exclude globs "*.md,*.json" + Then the repo-index result should not include file "README.md" + And the repo-index result should not include file "config.json" + + @feature195 + Scenario: Max total size truncates indexing + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files and a 2MB file + When I run repo-index full index with max total size 100 bytes + Then the repo-index result file count should be less than 6 + + # -- Removal ------------------------------------------------------------- + + @feature195 + Scenario: Remove index cleans up all records + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I remove the repo-index for the resource + Then querying repo-index for the resource should return nothing + + @feature195 + Scenario: Remove non-existent index returns false + Given a repo-index fresh in-memory indexing service + When I remove the repo-index for a non-existent resource + Then the repo-index removal result should be false + + # -- Error handling ------------------------------------------------------ + + @feature195 + Scenario: Index with empty resource ID raises error + Given a repo-index fresh in-memory indexing service + When I attempt repo-index with empty resource ID + Then a repo-index ValueError should be raised mentioning "non-empty" + + @feature195 + Scenario: Index with non-existent path raises error + Given a repo-index fresh in-memory indexing service + When I attempt repo-index with non-existent path "/nonexistent/xyz" + Then a repo-index FileNotFoundError should be raised + + @feature195 + Scenario: Index with invalid ULID resource ID raises error + Given a repo-index fresh in-memory indexing service + When I attempt repo-index with invalid ULID resource ID + Then a repo-index ValueError should be raised mentioning "valid ULID" + + # -- Status query --------------------------------------------------------- + + @feature195 + Scenario: Get index status returns metadata without file records + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I query repo-index status for the indexed resource + Then the repo-index status metadata should have status "ready" + And the repo-index status metadata file count should be 6 + + # -- Additional edge cases ----------------------------------------------- + + @feature195 + Scenario: Index with file path as root raises error + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I attempt repo-index with a file path as root + Then a repo-index ValueError should be raised mentioning "not a directory" + + @feature195 + Scenario: Index on empty directory returns zero files with ready status + Given a repo-index fresh in-memory indexing service + And a repo-index empty temporary directory + When I run repo-index full index on the sample directory + Then the repo-index result status should be "ready" + And the repo-index result file count should be 0 + And the repo-index primary language should be "unknown" + + # -- Refresh error paths ------------------------------------------------- + + @feature195 + Scenario: Refresh with non-existent path raises error + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I attempt repo-index refresh with non-existent path "/nonexistent/xyz" + Then a repo-index FileNotFoundError should be raised + + @feature195 + Scenario: Refresh with file path as root raises error + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I attempt repo-index refresh with a file path as root + Then a repo-index ValueError should be raised mentioning "not a directory" + + # -- Fault tolerance paths ----------------------------------------------- + + @feature195 + Scenario: Walk failure on fresh resource persists ERROR status + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When repo-index walk is mocked to fail and I index a fresh resource + Then a repo-index error should be raised + And querying repo-index status for the resource should show "error" + + @feature195 + Scenario: Walk failure on existing resource preserves previous index + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When repo-index walk is mocked to fail and I re-index + Then a repo-index error should be raised + And the repo-index previous index should still be retrievable with 6 files + + @feature195 + Scenario: Refresh walk failure preserves previous good index + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When repo-index walk is mocked to fail and I refresh + Then a repo-index error should be raised + And the repo-index previous index should still be retrievable with 6 files + + # -- Stale indexing cleanup ------------------------------------------------ + + @feature195 + Scenario: Cleanup stale indexing removes orphan INDEXING rows + Given a repo-index fresh in-memory indexing service + And a repo-index stale INDEXING row is inserted for a resource + When I call repo-index cleanup stale indexing + Then the repo-index cleanup should report 1 stale row removed + And querying repo-index for the stale resource should return nothing + + # -- Edge cases: unreadable files, naive datetimes ----------------------- + + @feature195 + Scenario: Unreadable file is skipped without crashing index + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with a file that fails hashing + When I run repo-index full index with hash failure on "secret.dat" + Then the repo-index result status should be "ready" + And the repo-index result should not include file "secret.dat" + + @feature195 + Scenario: Re-index replaces existing index completely + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + And I run repo-index full index on the sample directory + When I add a new file "extra.py" and re-index + Then the repo-index result file count should be 7 + And the repo-index result status should be "ready" + + @feature195 + Scenario: Naive datetime is coerced to UTC in FileRecord + Given a repo-index fresh in-memory indexing service + When I create a FileRecord with a naive datetime + Then the FileRecord last_modified should have UTC timezone + + @feature195 + Scenario: Naive datetime is coerced to UTC in IndexMetadata + Given a repo-index fresh in-memory indexing service + When I create an IndexMetadata with a naive datetime + Then the IndexMetadata indexed_at should have UTC timezone + + # -- Domain model structural assertions ---------------------------------- + + @feature195 + Scenario: IndexMetadata has all spec-required fields + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index on the sample directory + Then the repo-index metadata should have field "index_id" + And the repo-index metadata should have field "resource_id" + And the repo-index metadata should have field "indexed_at" + And the repo-index metadata should have field "file_count" + And the repo-index metadata should have field "token_estimate" + And the repo-index metadata should have field "primary_language" + And the repo-index metadata should have field "status" + + @feature195 + Scenario: FileRecord has all spec-required fields + Given a repo-index fresh in-memory indexing service + And a repo-index temporary directory with sample project files + When I run repo-index full index on the sample directory + Then each repo-index file record should have field "path" + And each repo-index file record should have field "content_hash" + And each repo-index file record should have field "token_count" + And each repo-index file record should have field "language" + And each repo-index file record should have field "size_bytes" + And each repo-index file record should have field "last_modified" diff --git a/features/steps/domain_analyzer_steps.py b/features/steps/domain_analyzer_steps.py index ee4b3478f..c57d7c3b1 100644 --- a/features/steps/domain_analyzer_steps.py +++ b/features/steps/domain_analyzer_steps.py @@ -267,7 +267,8 @@ def step_then_triple_pred_obj_uri_contains( context: Context, pred: str, fragment: str ) -> None: found = any( - t.predicate == pred and fragment in t.object_uri for t in context.triples + t.predicate == pred and t.object_uri is not None and fragment in t.object_uri + for t in context.triples ) assert found, ( f"No triple with predicate={pred!r} and object_uri containing " @@ -286,7 +287,10 @@ def step_then_triple_links_subject_to_object( found = any( t.predicate == pred and s_frag in t.subject_uri - and (o_frag in t.object_uri or o_frag in t.object_value) + and ( + (t.object_uri is not None and o_frag in t.object_uri) + or o_frag in t.object_value + ) for t in context.triples ) assert found, ( diff --git a/features/steps/repo_indexing_steps.py b/features/steps/repo_indexing_steps.py new file mode 100644 index 000000000..84934bc04 --- /dev/null +++ b/features/steps/repo_indexing_steps.py @@ -0,0 +1,732 @@ +"""Step definitions for features/repo_indexing.feature. + +Tests for the repository indexing service (issue #195). All context +attributes use the ``repo_index_`` prefix to avoid collisions with +other step files. +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +from behave import given, then, when # type: ignore[import-untyped] +from behave.runner import Context # type: ignore[import-untyped] +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from cleveragents.application.services.repo_indexing_service import ( + RepoIndexingService, +) +from cleveragents.domain.models.core.repo_index import ( + FileRecord, + IndexMetadata, + IndexStatus, + RepoIndex, +) +from cleveragents.infrastructure.database.models import Base, RepoIndexModel + +_RESOURCE_ID = "01KK0D8WNATFNEX2JMG5GKF6FM" + + +# ── Helpers ────────────────────────────────────────────────────────── + + +def _make_service(context: Context) -> RepoIndexingService: + """Create an in-memory RepoIndexingService and store on context.""" + engine = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + service = RepoIndexingService(session_factory=factory) + context.repo_index_service = service # type: ignore[attr-defined] + return service + + +def _make_sample_project(context: Context) -> Path: + """Create a temp directory with sample project files.""" + tmpdir = Path(tempfile.mkdtemp()) + (tmpdir / "main.py").write_text("print('hello world')\ndef main():\n pass\n") + (tmpdir / "utils.py").write_text("def helper():\n return 42\n") + (tmpdir / "index.ts").write_text("export const app = () => {};\n") + (tmpdir / "README.md").write_text("# Sample Project\n\nA test project.\n") + (tmpdir / "config.json").write_text('{"key": "value", "debug": true}\n') + (tmpdir / "Makefile").write_text("all:\n\t@echo hello\n") + context.repo_index_tmpdir = str(tmpdir) # type: ignore[attr-defined] + context.add_cleanup(_cleanup_tmpdir, context) + return tmpdir + + +def _cleanup_tmpdir(context: Context) -> None: + """Remove the temporary directory.""" + tmpdir = getattr(context, "repo_index_tmpdir", None) + if tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +# ── Given steps ────────────────────────────────────────────────────── + + +@given("a repo-index fresh in-memory indexing service") +def step_fresh_service(context: Context) -> None: + """Set up a fresh in-memory RepoIndexingService.""" + _make_service(context) + context.repo_index_result = None # type: ignore[attr-defined] + context.repo_index_original_result = None # type: ignore[attr-defined] + context.repo_index_error = None # type: ignore[attr-defined] + context.repo_index_removal_result = None # type: ignore[attr-defined] + + +@given("a repo-index temporary directory with sample project files") +def step_sample_project(context: Context) -> None: + """Create a temporary directory with sample files.""" + _make_sample_project(context) + + +@given("a repo-index temporary directory with sample project files and a 2MB file") +def step_sample_project_with_big_file(context: Context) -> None: + """Create a temporary directory with sample files plus a 2MB file.""" + tmpdir = _make_sample_project(context) + (tmpdir / "big_file.dat").write_bytes(b"x" * 2_000_000) + + +@given("a repo-index empty temporary directory") +def step_empty_directory(context: Context) -> None: + """Create an empty temporary directory.""" + tmpdir = Path(tempfile.mkdtemp()) + context.repo_index_tmpdir = str(tmpdir) # type: ignore[attr-defined] + context.add_cleanup(_cleanup_tmpdir, context) + + +@given("a repo-index temporary directory with a file that fails hashing") +def step_dir_with_unhashable_file(context: Context) -> None: + """Create a temp directory with a file that will fail during hashing. + + We create a normal file here; the actual OSError is injected via + ``unittest.mock.patch`` in the When step. This avoids reliance on + ``chmod 0o000`` which is ineffective when running as root. + """ + tmpdir = Path(tempfile.mkdtemp()) + (tmpdir / "visible.py").write_text("x = 1\n") + (tmpdir / "secret.dat").write_text("top secret\n") + context.repo_index_tmpdir = str(tmpdir) # type: ignore[attr-defined] + context.add_cleanup(_cleanup_tmpdir, context) + + +@given("I run repo-index full index on the sample directory") +def step_run_full_index_given(context: Context) -> None: + """Run a full index (as a Given precondition).""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.index_resource(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + context.repo_index_result = result # type: ignore[attr-defined] + context.repo_index_original_result = result # type: ignore[attr-defined] + + +# ── When steps ─────────────────────────────────────────────────────── + + +@when("I run repo-index full index on the sample directory") +def step_run_full_index(context: Context) -> None: + """Run a full index.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.index_resource(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + context.repo_index_result = result # type: ignore[attr-defined] + context.repo_index_original_result = result # type: ignore[attr-defined] + + +@when('I run repo-index full index with hash failure on "{filename}"') +def step_run_full_index_with_hash_failure(context: Context, filename: str) -> None: + """Run a full index where ``read_and_hash`` raises OSError for *filename*. + + This simulates an unreadable file without relying on ``chmod 0o000`` + (which is ineffective when the test runner is root). + We patch ``read_and_hash`` in the utils module where + ``walk_and_index`` actually calls it. + """ + import cleveragents.application.services.repo_indexing_utils as utils_mod + + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + original_read_and_hash = utils_mod.read_and_hash + + def _failing_read(path: Path) -> tuple[str, int]: + if Path(path).name == filename: + raise OSError(f"Simulated read failure for {filename}") + return original_read_and_hash(path) + + with patch.object(utils_mod, "read_and_hash", _failing_read): + result = service.index_resource( + _RESOURCE_ID, + context.repo_index_tmpdir, # type: ignore[attr-defined] + ) + context.repo_index_result = result # type: ignore[attr-defined] + context.repo_index_original_result = result # type: ignore[attr-defined] + + +@when("I query repo-index for the indexed resource") +def step_query_index(context: Context) -> None: + """Query the persisted index.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + context.repo_index_retrieved = service.get_index(_RESOURCE_ID) # type: ignore[attr-defined] + + +@when('I modify repo-index file "{filename}" with new content') +def step_modify_file(context: Context, filename: str) -> None: + """Modify a file in the temp directory.""" + filepath = Path(context.repo_index_tmpdir) / filename # type: ignore[attr-defined] + filepath.write_text("# Modified content\nprint('updated')\n" * 5) + + +@when("I run repo-index incremental refresh") +def step_run_refresh(context: Context) -> None: + """Run incremental refresh.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.refresh_index(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + context.repo_index_result = result # type: ignore[attr-defined] + + +@when("I run repo-index full index with max file size {size:d} bytes") +def step_run_index_max_file_size(context: Context, size: int) -> None: + """Run index with max_file_size policy.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.index_resource( + _RESOURCE_ID, + context.repo_index_tmpdir, # type: ignore[attr-defined] + max_file_size=size, + ) + context.repo_index_result = result # type: ignore[attr-defined] + + +@when('I run repo-index full index with include globs "{globs}"') +def step_run_index_include_globs(context: Context, globs: str) -> None: + """Run index with include globs.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + glob_tuple = tuple(g.strip() for g in globs.split(",")) + result = service.index_resource( + _RESOURCE_ID, + context.repo_index_tmpdir, # type: ignore[attr-defined] + include_globs=glob_tuple, + ) + context.repo_index_result = result # type: ignore[attr-defined] + + +@when('I run repo-index full index with exclude globs "{globs}"') +def step_run_index_exclude_globs(context: Context, globs: str) -> None: + """Run index with exclude globs.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + glob_tuple = tuple(g.strip() for g in globs.split(",")) + result = service.index_resource( + _RESOURCE_ID, + context.repo_index_tmpdir, # type: ignore[attr-defined] + exclude_globs=glob_tuple, + ) + context.repo_index_result = result # type: ignore[attr-defined] + + +@when("I run repo-index full index with max total size {size:d} bytes") +def step_run_index_max_total_size(context: Context, size: int) -> None: + """Run index with max_total_size policy.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.index_resource( + _RESOURCE_ID, + context.repo_index_tmpdir, # type: ignore[attr-defined] + max_total_size=size, + ) + context.repo_index_result = result # type: ignore[attr-defined] + + +@when("I remove the repo-index for the resource") +def step_remove_index(context: Context) -> None: + """Remove the index.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + context.repo_index_removal_result = service.remove_index(_RESOURCE_ID) # type: ignore[attr-defined] + + +@when("I remove the repo-index for a non-existent resource") +def step_remove_nonexistent(context: Context) -> None: + """Attempt to remove an index that doesn't exist.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + context.repo_index_removal_result = service.remove_index( + "01KK0D8WNATFNEX2JMG5GKF6FN" + ) # type: ignore[attr-defined] + + +@when("I query repo-index status for the indexed resource") +def step_query_status(context: Context) -> None: + """Query the index status (metadata only, no file records).""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + context.repo_index_status = service.get_index_status(_RESOURCE_ID) # type: ignore[attr-defined] + + +@when("I attempt repo-index with a file path as root") +def step_index_file_root(context: Context) -> None: + """Attempt indexing with a file path (not a directory).""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + file_path = Path(context.repo_index_tmpdir) / "main.py" # type: ignore[attr-defined] + try: + service.index_resource(_RESOURCE_ID, str(file_path)) + except (ValueError, FileNotFoundError) as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when('I attempt repo-index refresh with non-existent path "{path}"') +def step_refresh_bad_path(context: Context, path: str) -> None: + """Attempt refresh with a non-existent path.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + try: + service.refresh_index(_RESOURCE_ID, path) + except FileNotFoundError as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when("I attempt repo-index refresh with a file path as root") +def step_refresh_file_root(context: Context) -> None: + """Attempt refresh with a file path (not a directory).""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + file_path = Path(context.repo_index_tmpdir) / "main.py" # type: ignore[attr-defined] + try: + service.refresh_index(_RESOURCE_ID, str(file_path)) + except (ValueError, FileNotFoundError) as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when("repo-index walk is mocked to fail and I index a fresh resource") +def step_index_mock_walk_fail_fresh(context: Context) -> None: + """Mock ``walk_and_index`` to raise OSError and attempt index. + + This tests the fault handler at lines 300-325 of the service: when a + walk failure occurs on a *fresh* resource (no prior index), the + service should persist an ERROR status row. + + We use ``unittest.mock.patch`` because ``os.walk`` silently skips + unreadable directories on Linux rather than raising. + """ + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + with patch( + "cleveragents.application.services.repo_indexing_service.walk_and_index", + side_effect=OSError("Simulated walk failure"), + ): + try: + service.index_resource(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + except Exception as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when("repo-index walk is mocked to fail and I re-index") +def step_reindex_mock_walk_fail(context: Context) -> None: + """Mock ``walk_and_index`` to raise OSError and attempt re-index. + + Tests the fault handler when a prior index exists: the service should + preserve the previous good index and re-raise. + """ + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + with patch( + "cleveragents.application.services.repo_indexing_service.walk_and_index", + side_effect=OSError("Simulated walk failure"), + ): + try: + service.index_resource(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + except Exception as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when("repo-index walk is mocked to fail and I refresh") +def step_refresh_mock_walk_fail(context: Context) -> None: + """Mock ``walk_and_index`` to raise OSError and attempt refresh. + + Tests the fault handler in ``refresh_index``: the service should + preserve the previous good index and re-raise. + """ + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + with patch( + "cleveragents.application.services.repo_indexing_service.walk_and_index", + side_effect=OSError("Simulated walk failure"), + ): + try: + service.refresh_index(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + except Exception as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when('I add a new file "{filename}" and re-index') +def step_add_file_reindex(context: Context, filename: str) -> None: + """Add a new file and re-index the same resource.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + filepath = Path(context.repo_index_tmpdir) / filename # type: ignore[attr-defined] + filepath.write_text("# New file\ndef new_func(): pass\n") + result = service.index_resource(_RESOURCE_ID, context.repo_index_tmpdir) # type: ignore[attr-defined] + context.repo_index_result = result # type: ignore[attr-defined] + + +@when("I create a FileRecord with a naive datetime") +def step_create_filerecord_naive(context: Context) -> None: + """Create a FileRecord with a naive (no tzinfo) datetime.""" + from datetime import datetime as dt + + naive = dt(2025, 6, 15, 12, 0, 0) + context.repo_index_filerecord = FileRecord( # type: ignore[attr-defined] + path="test.py", + content_hash="a" * 64, + token_count=10, + language="python", + size_bytes=40, + last_modified=naive, + ) + + +@when("I create an IndexMetadata with a naive datetime") +def step_create_metadata_naive(context: Context) -> None: + """Create an IndexMetadata with a naive (no tzinfo) datetime.""" + from datetime import datetime as dt + + naive = dt(2025, 6, 15, 12, 0, 0) + context.repo_index_metadata_naive = IndexMetadata( # type: ignore[attr-defined] + index_id="01KK0D8WNATFNEX2JMG5GKF6FM", + resource_id="01KK0D8WNATFNEX2JMG5GKF6FN", + indexed_at=naive, + file_count=0, + token_estimate=0, + primary_language="unknown", + status=IndexStatus.READY, + ) + + +@when("I attempt repo-index with invalid ULID resource ID") +def step_index_invalid_ulid(context: Context) -> None: + """Attempt indexing with an invalid ULID resource ID.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + try: + service.index_resource("not-a-valid-ulid", tempfile.gettempdir()) + except ValueError as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when("I attempt repo-index with empty resource ID") +def step_index_empty_id(context: Context) -> None: + """Attempt indexing with empty resource ID.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + try: + service.index_resource("", tempfile.gettempdir()) + except (ValueError, FileNotFoundError) as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +@when('I attempt repo-index with non-existent path "{path}"') +def step_index_bad_path(context: Context, path: str) -> None: + """Attempt indexing with a non-existent path.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + try: + service.index_resource(_RESOURCE_ID, path) + except FileNotFoundError as exc: + context.repo_index_error = exc # type: ignore[attr-defined] + + +# ── Then steps ─────────────────────────────────────────────────────── + + +@then('the repo-index result status should be "{status}"') +def step_check_status(context: Context, status: str) -> None: + """Assert the index result status.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + actual = result.metadata.status.value + assert actual == status, f"Expected status '{status}', got '{actual}'" + + +@then("the repo-index result file count should be {count:d}") +def step_check_file_count(context: Context, count: int) -> None: + """Assert the file count.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + actual = result.metadata.file_count + assert actual == count, ( + f"Expected {count} files, got {actual}. Files: {[f.path for f in result.files]}" + ) + + +@then("the repo-index result file count should be less than {count:d}") +def step_check_file_count_less(context: Context, count: int) -> None: + """Assert the file count is less than a value.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + actual = result.metadata.file_count + assert actual < count, f"Expected fewer than {count} files, got {actual}" + + +@then('the repo-index primary language should be "{lang}"') +def step_check_language(context: Context, lang: str) -> None: + """Assert the primary language.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + actual = result.metadata.primary_language + assert actual == lang, f"Expected language '{lang}', got '{actual}'" + + +@then("the repo-index token estimate should be positive") +def step_check_tokens_positive(context: Context) -> None: + """Assert token estimate is > 0.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + assert result.metadata.token_estimate > 0, ( + f"Expected positive token_estimate, got {result.metadata.token_estimate}" + ) + + +@then('the repo-index file "{filename}" should have language "{lang}"') +def step_check_file_language(context: Context, filename: str, lang: str) -> None: + """Assert a specific file's detected language.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + matches = [f for f in result.files if f.path == filename] + assert matches, ( + f"File '{filename}' not found in index. Files: {[f.path for f in result.files]}" + ) + actual = matches[0].language + assert actual == lang, f"Expected '{filename}' language='{lang}', got '{actual}'" + + +@then("the repo-index retrieved index should match the original") +def step_check_retrieved_matches(context: Context) -> None: + """Assert the retrieved index matches the original.""" + original: RepoIndex = context.repo_index_original_result # type: ignore[attr-defined] + retrieved: RepoIndex = context.repo_index_retrieved # type: ignore[attr-defined] + assert retrieved is not None, "Retrieved index is None" + assert retrieved.metadata.file_count == original.metadata.file_count + assert retrieved.metadata.token_estimate == original.metadata.token_estimate + assert retrieved.metadata.primary_language == original.metadata.primary_language + assert len(retrieved.files) == len(original.files) + + +@then("the repo-index retrieved file hashes should match the original") +def step_check_retrieved_hashes(context: Context) -> None: + """Assert the retrieved index has identical per-file content hashes.""" + original: RepoIndex = context.repo_index_original_result # type: ignore[attr-defined] + retrieved: RepoIndex = context.repo_index_retrieved # type: ignore[attr-defined] + orig_map = {f.path: f.content_hash for f in original.files} + retr_map = {f.path: f.content_hash for f in retrieved.files} + assert orig_map == retr_map, ( + f"File hash mismatch.\nOriginal: {orig_map}\nRetrieved: {retr_map}" + ) + + +@then('the repo-index file "{filename}" content hash should differ from original') +def step_check_hash_differs(context: Context, filename: str) -> None: + """Assert a file's hash changed after modification.""" + original: RepoIndex = context.repo_index_original_result # type: ignore[attr-defined] + current: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + orig_hash = next( + (f.content_hash for f in original.files if f.path == filename), + None, + ) + curr_hash = next( + (f.content_hash for f in current.files if f.path == filename), + None, + ) + assert orig_hash is not None, f"'{filename}' not found in original index" + assert curr_hash is not None, f"'{filename}' not found in current index" + assert orig_hash != curr_hash, ( + f"Expected hash for '{filename}' to change, but both are '{orig_hash}'" + ) + + +@then('the repo-index file "{filename}" content hash should match original') +def step_check_hash_matches(context: Context, filename: str) -> None: + """Assert a file's hash is unchanged.""" + original: RepoIndex = context.repo_index_original_result # type: ignore[attr-defined] + current: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + orig_hash = next( + (f.content_hash for f in original.files if f.path == filename), + None, + ) + curr_hash = next( + (f.content_hash for f in current.files if f.path == filename), + None, + ) + assert orig_hash is not None, f"'{filename}' not found in original index" + assert curr_hash is not None, f"'{filename}' not found in current index" + assert orig_hash == curr_hash, ( + f"Expected hash for '{filename}' to match, but got '{orig_hash}' vs '{curr_hash}'" + ) + + +@then('the repo-index result should not include file "{filename}"') +def step_check_file_excluded(context: Context, filename: str) -> None: + """Assert a file is not in the index.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + paths = [f.path for f in result.files] + assert filename not in paths, f"File '{filename}' should be excluded but was found" + + +@then("a repo-index error should be raised") +def step_check_any_error(context: Context) -> None: + """Assert some error was raised.""" + err = context.repo_index_error # type: ignore[attr-defined] + assert err is not None, "Expected an error but none raised" + + +@then('querying repo-index status for the resource should show "{status}"') +def step_check_error_status_persisted(context: Context, status: str) -> None: + """Assert the persisted index status matches.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + meta = service.get_index_status(_RESOURCE_ID) + assert meta is not None, ( + f"Expected persisted status '{status}' but get_index_status returned None" + ) + actual = meta.status.value + assert actual == status, f"Expected status '{status}', got '{actual}'" + + +@then("the repo-index previous index should still be retrievable with {count:d} files") +def step_check_previous_index(context: Context, count: int) -> None: + """Assert the previous index is still intact.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + idx = service.get_index(_RESOURCE_ID) + assert idx is not None, "Previous index was lost" + assert idx.metadata.file_count == count, ( + f"Expected {count} files, got {idx.metadata.file_count}" + ) + + +@then("the FileRecord last_modified should have UTC timezone") +def step_check_filerecord_utc(context: Context) -> None: + """Assert the FileRecord has UTC timezone on last_modified.""" + from datetime import UTC + + fr: FileRecord = context.repo_index_filerecord # type: ignore[attr-defined] + assert fr.last_modified.tzinfo is not None, "last_modified has no timezone" + assert fr.last_modified.tzinfo == UTC, ( + f"Expected UTC, got {fr.last_modified.tzinfo}" + ) + + +@then("the IndexMetadata indexed_at should have UTC timezone") +def step_check_metadata_utc(context: Context) -> None: + """Assert the IndexMetadata has UTC timezone on indexed_at.""" + from datetime import UTC + + meta: IndexMetadata = context.repo_index_metadata_naive # type: ignore[attr-defined] + assert meta.indexed_at.tzinfo is not None, "indexed_at has no timezone" + assert meta.indexed_at.tzinfo == UTC, f"Expected UTC, got {meta.indexed_at.tzinfo}" + + +@then('the repo-index status metadata should have status "{status}"') +def step_check_status_metadata(context: Context, status: str) -> None: + """Assert the status metadata has the expected status.""" + + meta: IndexMetadata = context.repo_index_status # type: ignore[attr-defined] + assert meta is not None, "get_index_status returned None" + actual = meta.status.value + assert actual == status, f"Expected status '{status}', got '{actual}'" + + +@then("the repo-index status metadata file count should be {count:d}") +def step_check_status_file_count(context: Context, count: int) -> None: + """Assert the status metadata file count.""" + + meta: IndexMetadata = context.repo_index_status # type: ignore[attr-defined] + assert meta is not None, "get_index_status returned None" + actual = meta.file_count + assert actual == count, f"Expected {count} files, got {actual}" + + +@then("querying repo-index for the resource should return nothing") +def step_check_removed(context: Context) -> None: + """Assert the index is gone.""" + service: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + result = service.get_index(_RESOURCE_ID) + assert result is None, f"Expected None after removal, got {result}" + + +@then("the repo-index removal result should be false") +def step_check_removal_false(context: Context) -> None: + """Assert removal returned False.""" + assert context.repo_index_removal_result is False # type: ignore[attr-defined] + + +@then('a repo-index ValueError should be raised mentioning "{text}"') +def step_check_value_error(context: Context, text: str) -> None: + """Assert a ValueError was raised with expected message.""" + err = context.repo_index_error # type: ignore[attr-defined] + assert err is not None, "Expected ValueError but none raised" + assert isinstance(err, ValueError), f"Expected ValueError, got {type(err).__name__}" + assert text in str(err), f"Expected '{text}' in error: {err}" + + +@then("a repo-index FileNotFoundError should be raised") +def step_check_fnf_error(context: Context) -> None: + """Assert a FileNotFoundError was raised.""" + err = context.repo_index_error # type: ignore[attr-defined] + assert err is not None, "Expected FileNotFoundError but none raised" + assert isinstance(err, FileNotFoundError), ( + f"Expected FileNotFoundError, got {type(err).__name__}" + ) + + +@then('the repo-index metadata should have field "{field}"') +def step_check_metadata_field(context: Context, field: str) -> None: + """Assert the metadata has a specific field.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + assert hasattr(result.metadata, field), ( + f"IndexMetadata missing field '{field}'. " + f"Available: {list(result.metadata.model_fields.keys())}" + ) + + +@then('each repo-index file record should have field "{field}"') +def step_check_file_field(context: Context, field: str) -> None: + """Assert every file record has a specific field.""" + result: RepoIndex = context.repo_index_result # type: ignore[attr-defined] + assert len(result.files) > 0, "No file records to check" + for fr in result.files: + assert hasattr(fr, field), ( + f"FileRecord missing field '{field}'. " + f"Available: {list(fr.model_fields.keys())}" + ) + + +# ── Stale indexing cleanup steps ───────────────────────────────────── + + +_STALE_RESOURCE_ID = "01KK0D8WNATFNEX2JMG5GKF6FN" + + +@given("a repo-index stale INDEXING row is inserted for a resource") +def step_insert_stale_indexing_row(context: Context) -> None: + """Manually insert an INDEXING-status row to simulate a crash.""" + from datetime import UTC, datetime + + svc: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + session = svc._session_factory() + try: + row = RepoIndexModel( + index_id="01KK0D8WNATFNEX2JMG5GKF6FZ", + resource_id=_STALE_RESOURCE_ID, + file_count=0, + token_estimate=0, + primary_language="unknown", + status="indexing", + indexed_at=datetime.now(tz=UTC).isoformat(), + created_at=datetime.now(tz=UTC).isoformat(), + ) + session.add(row) + session.commit() + finally: + session.close() + context.repo_index_stale_resource_id = _STALE_RESOURCE_ID # type: ignore[attr-defined] + + +@when("I call repo-index cleanup stale indexing") +def step_call_cleanup(context: Context) -> None: + """Call cleanup_stale_indexing on the service.""" + svc: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + context.repo_index_cleanup_count = svc.cleanup_stale_indexing() # type: ignore[attr-defined] + + +@then("the repo-index cleanup should report {count:d} stale row removed") +def step_check_cleanup_count(context: Context, count: int) -> None: + """Assert the cleanup returned the expected count.""" + actual = context.repo_index_cleanup_count # type: ignore[attr-defined] + assert actual == count, f"Expected {count} stale rows, got {actual}" + + +@then("querying repo-index for the stale resource should return nothing") +def step_check_stale_removed(context: Context) -> None: + """Assert the stale row no longer exists.""" + svc: RepoIndexingService = context.repo_index_service # type: ignore[attr-defined] + rid = context.repo_index_stale_resource_id # type: ignore[attr-defined] + result = svc.get_index(rid) + assert result is None, f"Expected None, got {result}" diff --git a/robot/domain_analyzers.robot b/robot/domain_analyzers.robot index 24c4d7101..bc048c3ce 100644 --- a/robot/domain_analyzers.robot +++ b/robot/domain_analyzers.robot @@ -1,7 +1,8 @@ *** Settings *** Documentation Integration smoke tests for domain-specific analyzers -Library Process -Suite Setup Log Domain Analyzers Robot Tests +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment *** Variables *** ${HELPER} ${CURDIR}/helper_domain_analyzers.py @@ -9,42 +10,48 @@ ${HELPER} ${CURDIR}/helper_domain_analyzers.py *** Test Cases *** Analyze Python Source [Documentation] PythonAnalyzer extracts Module, Class, and Function triples - ${result}= Run Process python3 ${HELPER} analyze-python + ${result}= Run Process ${PYTHON} ${HELPER} analyze-python timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} analyze-python-ok Analyze Markdown Document [Documentation] MarkdownAnalyzer extracts Document and Section triples - ${result}= Run Process python3 ${HELPER} analyze-markdown + ${result}= Run Process ${PYTHON} ${HELPER} analyze-markdown timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} analyze-markdown-ok Analyze PostgreSQL DDL [Documentation] PostgreSQLAnalyzer extracts Table and Column triples - ${result}= Run Process python3 ${HELPER} analyze-postgresql + ${result}= Run Process ${PYTHON} ${HELPER} analyze-postgresql timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} analyze-postgresql-ok Analyze Docker Compose YAML [Documentation] DockerComposeAnalyzer extracts Service and DeploymentUnit triples - ${result}= Run Process python3 ${HELPER} analyze-docker-compose + ${result}= Run Process ${PYTHON} ${HELPER} analyze-docker-compose timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} analyze-docker-compose-ok Verify Analyzer Protocol Conformance [Documentation] All four analyzers satisfy AnalyzerProtocol - ${result}= Run Process python3 ${HELPER} protocol-check + ${result}= Run Process ${PYTHON} ${HELPER} protocol-check timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} protocol-check-ok Verify Analyzer Registry Lookup [Documentation] AnalyzerRegistry registers all four analyzers and resolves by extension - ${result}= Run Process python3 ${HELPER} registry-check + ${result}= Run Process ${PYTHON} ${HELPER} registry-check timeout=30s Log ${result.stdout} + Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} registry-check-ok diff --git a/robot/helper_repo_indexing.py b/robot/helper_repo_indexing.py new file mode 100644 index 000000000..26b0466d3 --- /dev/null +++ b/robot/helper_repo_indexing.py @@ -0,0 +1,193 @@ +"""Robot Framework helper for repo indexing integration tests. + +Verifies that the RepoIndexingService can index a directory, refresh +incrementally, enforce policy limits, and clean up on removal. + +Usage: + python robot/helper_repo_indexing.py full-index + python robot/helper_repo_indexing.py incremental-refresh + python robot/helper_repo_indexing.py policy-enforcement +""" + +from __future__ import annotations + +import shutil +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from sqlalchemy import create_engine # noqa: E402 +from sqlalchemy.orm import sessionmaker # noqa: E402 + +from cleveragents.application.services.repo_indexing_service import ( # noqa: E402 + RepoIndexingService, +) +from cleveragents.infrastructure.database.models import Base # noqa: E402 + +_RID = "01KK0D8WNATFNEX2JMG5GKF6FQ" + + +def _make_service() -> RepoIndexingService: + engine = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + return RepoIndexingService(session_factory=factory) + + +def _make_sample_dir() -> str: + tmpdir = tempfile.mkdtemp() + (Path(tmpdir) / "app.py").write_text("print('hello')\n") + (Path(tmpdir) / "lib.py").write_text("def f(): return 1\n") + (Path(tmpdir) / "README.md").write_text("# App\n") + return tmpdir + + +def _cmd_full_index() -> int: + svc = _make_service() + tmpdir = _make_sample_dir() + try: + idx = svc.index_resource(_RID, tmpdir) + if idx.metadata.status.value != "ready": + print(f"FAIL: status={idx.metadata.status.value}") + return 1 + if idx.metadata.file_count != 3: + print(f"FAIL: file_count={idx.metadata.file_count}") + return 1 + if idx.metadata.primary_language != "python": + print(f"FAIL: language={idx.metadata.primary_language}") + return 1 + + # Verify persistence + retrieved = svc.get_index(_RID) + if retrieved is None: + print("FAIL: get_index returned None") + return 1 + if retrieved.metadata.file_count != 3: + print(f"FAIL: retrieved file_count={retrieved.metadata.file_count}") + return 1 + + print("repo-indexing-full-ok") + return 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _cmd_incremental_refresh() -> int: + svc = _make_service() + tmpdir = _make_sample_dir() + try: + # Initial index + original = svc.index_resource(_RID, tmpdir) + orig_hash = next( + (f.content_hash for f in original.files if f.path == "app.py"), + None, + ) + if orig_hash is None: + print("FAIL: app.py not found in original index") + return 1 + + # Modify a file + (Path(tmpdir) / "app.py").write_text("print('updated')\n# more\n") + + # Refresh + refreshed = svc.refresh_index(_RID, tmpdir) + new_hash = next( + (f.content_hash for f in refreshed.files if f.path == "app.py"), + None, + ) + if new_hash is None: + print("FAIL: app.py not found in refreshed index") + return 1 + + if orig_hash == new_hash: + print("FAIL: hash should have changed") + return 1 + + # Unchanged file + orig_lib = next( + (f.content_hash for f in original.files if f.path == "lib.py"), + None, + ) + new_lib = next( + (f.content_hash for f in refreshed.files if f.path == "lib.py"), + None, + ) + if orig_lib is None or new_lib is None: + print("FAIL: lib.py not found in index") + return 1 + if orig_lib != new_lib: + print("FAIL: unchanged file hash changed") + return 1 + + print("repo-indexing-refresh-ok") + return 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _cmd_policy_enforcement() -> int: + svc = _make_service() + tmpdir = _make_sample_dir() + try: + (Path(tmpdir) / "big.dat").write_bytes(b"x" * 2_000_000) + # Max file size + idx = svc.index_resource( + "01KK0D8WNATFNEX2JMG5GKF6FR", tmpdir, max_file_size=1_000_000 + ) + paths = [f.path for f in idx.files] + if "big.dat" in paths: + print("FAIL: big.dat should be excluded by max_file_size") + return 1 + + # Include globs + idx2 = svc.index_resource( + "01KK0DAE56DZVRZKRHA53K1MQG", tmpdir, include_globs=("*.py",) + ) + if idx2.metadata.file_count != 2: + print(f"FAIL: include_globs *.py got {idx2.metadata.file_count} files") + return 1 + + # Removal + removed = svc.remove_index("01KK0D8WNATFNEX2JMG5GKF6FR") + if not removed: + print("FAIL: remove_index returned False") + return 1 + if svc.get_index("01KK0D8WNATFNEX2JMG5GKF6FR") is not None: + print("FAIL: index still exists after removal") + return 1 + + print("repo-indexing-policy-ok") + return 0 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +_COMMANDS: dict[str, Callable[[], int]] = { + "full-index": _cmd_full_index, + "incremental-refresh": _cmd_incremental_refresh, + "policy-enforcement": _cmd_policy_enforcement, +} + + +def main() -> int: + if len(sys.argv) < 2: + print( + "Usage: helper_repo_indexing.py " + "" + ) + return 1 + command = sys.argv[1] + handler = _COMMANDS.get(command) + if handler is None: + print(f"Unknown command: {command}") + return 1 + return handler() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/robot/repo_indexing.robot b/robot/repo_indexing.robot new file mode 100644 index 000000000..89ec9ba03 --- /dev/null +++ b/robot/repo_indexing.robot @@ -0,0 +1,37 @@ +*** Settings *** +Documentation Integration tests for repo indexing service (issue #195). +... Verifies full indexing, incremental refresh, and policy enforcement. +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_repo_indexing.py + +*** Test Cases *** +Full Index With Language Detection And Persistence + [Documentation] Index a sample directory, verify file count, language, and persistence. + [Tags] feature195 + ${result}= Run Process ${PYTHON} ${HELPER} full-index cwd=${WORKSPACE} timeout=30s + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} repo-indexing-full-ok + +Incremental Refresh Detects Changed Files + [Documentation] Modify a file and verify incremental refresh detects the change. + [Tags] feature195 + ${result}= Run Process ${PYTHON} ${HELPER} incremental-refresh cwd=${WORKSPACE} timeout=30s + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} repo-indexing-refresh-ok + +Policy Enforcement And Index Removal + [Documentation] Verify max_file_size, include_globs, and removal. + [Tags] feature195 + ${result}= Run Process ${PYTHON} ${HELPER} policy-enforcement cwd=${WORKSPACE} timeout=30s + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} repo-indexing-policy-ok diff --git a/src/cleveragents/application/container.py b/src/cleveragents/application/container.py index dcc4572f1..0da4e202b 100644 --- a/src/cleveragents/application/container.py +++ b/src/cleveragents/application/container.py @@ -37,6 +37,9 @@ from cleveragents.application.services.plan_lifecycle_service import ( ) from cleveragents.application.services.plan_service import PlanService from cleveragents.application.services.project_service import ProjectService +from cleveragents.application.services.repo_indexing_service import ( + RepoIndexingService, +) from cleveragents.application.services.resource_registry_service import ( ResourceRegistryService, ) @@ -138,6 +141,26 @@ def get_database_url() -> str: return f"sqlite:///{db_path.absolute()}" +def _build_repo_indexing_service( + database_url: str, +) -> RepoIndexingService: + """Build a RepoIndexingService with a session factory from the database URL. + + Factory-per-resolution matches the existing project pattern (see + _build_resource_registry_service, _build_namespaced_project_repo, etc.). + + Note: SQLite FK ON DELETE CASCADE requires ``PRAGMA foreign_keys=ON`` + per connection. We rely on manual child-row deletion in persistence + helpers instead, keeping consistency with the rest of the codebase. + """ + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + engine = create_engine(database_url, echo=False) + factory = sessionmaker(bind=engine, expire_on_commit=False) + return RepoIndexingService(session_factory=factory) + + def _build_resource_registry_service( database_url: str, ) -> ResourceRegistryService: @@ -327,6 +350,12 @@ class Container(containers.DeclarativeContainer): decision_service=decision_service, ) + # Repo Indexing Service - indexes repository resources (#195) + repo_indexing_service = providers.Factory( + _build_repo_indexing_service, + database_url=database_url, + ) + # Resource Registry Service - uses database session factory from UoW resource_registry_service = providers.Factory( _build_resource_registry_service, diff --git a/src/cleveragents/application/services/__init__.py b/src/cleveragents/application/services/__init__.py index 61a7f8259..65089c877 100644 --- a/src/cleveragents/application/services/__init__.py +++ b/src/cleveragents/application/services/__init__.py @@ -93,6 +93,9 @@ from cleveragents.application.services.prompt_sanitizer import ( PromptSanitizer, SanitizationResult, ) +from cleveragents.application.services.repo_indexing_service import ( + RepoIndexingService, +) from cleveragents.application.services.semantic_validation_rules import ( APIMisuseRule, BrokenReferenceRule, @@ -230,6 +233,7 @@ __all__ = [ "PromptSanitizer", "ProvenancePreambleGenerator", "RelevanceCoherenceOrderer", + "RepoIndexingService", "ResolvedValue", "RuntimeExecuteActor", "RuntimeExecuteResult", diff --git a/src/cleveragents/application/services/repo_indexing_persistence.py b/src/cleveragents/application/services/repo_indexing_persistence.py new file mode 100644 index 000000000..9793935ff --- /dev/null +++ b/src/cleveragents/application/services/repo_indexing_persistence.py @@ -0,0 +1,276 @@ +"""Database persistence helpers for repository indexing. + +Extracted from :mod:`~cleveragents.application.services.repo_indexing_service` +to keep the service module under the 500-line limit (CONTRIBUTING.md line 396). + +Functions accept a ``session_factory`` callable (matching the DI pattern +used elsewhere in the codebase) and handle session lifecycle internally. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, cast + +from cleveragents.domain.models.core.repo_index import ( + FileRecord, + IndexMetadata, + IndexStatus, + RepoIndex, +) +from cleveragents.infrastructure.database.models import ( + IndexedFileModel, + RepoIndexModel, +) + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +__all__ = [ + "load_index", + "load_index_status", + "persist_index", + "persist_status", +] + +logger = logging.getLogger(__name__) + +# Batch size for bulk file-record inserts (G4 fix). +_INSERT_BATCH_SIZE = 500 + + +def _safe_fromisoformat(value: str) -> datetime: + """Parse an ISO-8601 timestamp, falling back to UTC now on error (G3 fix). + + Returned datetime is always UTC-aware: naive results get UTC attached, + and aware non-UTC datetimes are converted via ``astimezone(UTC)``. + """ + try: + dt = datetime.fromisoformat(value) + except (ValueError, TypeError): + logger.warning( + "Corrupt timestamp in database, using UTC now as fallback", + extra={"raw_value": value[:80] if value else ""}, + ) + return datetime.now(tz=UTC) + if dt.tzinfo is None: + return dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +def _session(session_factory: Any) -> Session: + """Obtain a database session from the factory.""" + return session_factory() + + +def persist_status( + session_factory: Any, + *, + index_id: str, + resource_id: str, + status: IndexStatus, + error_message: str | None = None, +) -> None: + """Persist or update just the index status row (no file records). + + Used to record ``INDEXING`` before the walk starts and ``ERROR`` + on failure, ensuring the lifecycle is observable. + """ + if error_message is not None and status != IndexStatus.ERROR: + raise ValueError("error_message must be None when status is not ERROR") + + session = _session(session_factory) + try: + now_iso = datetime.now(tz=UTC).isoformat() + + existing = ( + session.query(RepoIndexModel).filter_by(resource_id=resource_id).first() + ) + if existing is not None: + # Manual child-row deletion required because SQLite does not + # honour FK ON DELETE CASCADE without PRAGMA foreign_keys=ON. + session.query(IndexedFileModel).filter_by( + index_id=existing.index_id + ).delete() + session.delete(existing) + session.flush() + + db_index = RepoIndexModel( + index_id=index_id, + resource_id=resource_id, + file_count=0, + token_estimate=0, + primary_language="unknown", + status=status.value, + error_message=error_message, + indexed_at=now_iso, + created_at=now_iso, + ) + session.add(db_index) + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + +def persist_index(session_factory: Any, repo_index: RepoIndex) -> None: + """Persist a RepoIndex to the database (upsert semantics). + + Deletes any existing index for the same resource_id, then inserts + the new index metadata and all file records in bulk. + """ + meta = repo_index.metadata + session = _session(session_factory) + try: + # Remove existing index for this resource then insert fresh rows + # (delete-insert instead of in-place update so that the row set + # exactly matches the new file list without orphan cleanup). + existing = ( + session.query(RepoIndexModel) + .filter_by(resource_id=meta.resource_id) + .first() + ) + if existing is not None: + # Manual child-row deletion — see persist_status comment. + session.query(IndexedFileModel).filter_by( + index_id=existing.index_id + ).delete() + session.delete(existing) + session.flush() + + now_iso = datetime.now(tz=UTC).isoformat() + + # Insert index metadata + db_index = RepoIndexModel( + index_id=meta.index_id, + resource_id=meta.resource_id, + file_count=meta.file_count, + token_estimate=meta.token_estimate, + primary_language=meta.primary_language, + status=meta.status.value, + error_message=meta.error_message, + indexed_at=meta.indexed_at.isoformat(), + created_at=now_iso, + ) + session.add(db_index) + session.flush() + + # Bulk insert file records in batches to bound memory (G4 fix). + files = repo_index.files + for batch_start in range(0, len(files), _INSERT_BATCH_SIZE): + batch = files[batch_start : batch_start + _INSERT_BATCH_SIZE] + file_models = [ + IndexedFileModel( + index_id=meta.index_id, + path=fr.path, + content_hash=fr.content_hash, + token_count=fr.token_count, + size_bytes=fr.size_bytes, + language=fr.language, + last_modified=fr.last_modified.isoformat(), + ) + for fr in batch + ] + session.add_all(file_models) + session.flush() + + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + +def load_index(session_factory: Any, resource_id: str) -> RepoIndex | None: + """Load a full RepoIndex (metadata + files) from the database. + + Returns: + The :class:`RepoIndex` if it exists, or ``None``. + """ + session = _session(session_factory) + try: + row = session.query(RepoIndexModel).filter_by(resource_id=resource_id).first() + if row is None: + return None + + file_rows = ( + session.query(IndexedFileModel).filter_by(index_id=row.index_id).all() + ) + + # Build records individually; skip corrupt rows rather than + # failing the entire load (defensive against DB corruption). + file_list: list[FileRecord] = [] + for fr in file_rows: + try: + file_list.append( + FileRecord( + path=cast(str, fr.path), + content_hash=cast(str, fr.content_hash), + token_count=cast(int, fr.token_count), + language=cast(str, fr.language), + size_bytes=cast(int, fr.size_bytes), + last_modified=_safe_fromisoformat(cast(str, fr.last_modified)), + ) + ) + except (ValueError, TypeError): + logger.warning( + "Skipping corrupt file record", + extra={"path": getattr(fr, "path", "?")}, + ) + files = tuple(file_list) + + # N13 fix: treat both None and empty string as None. + raw_error = cast(str | None, row.error_message) + error_msg = raw_error if raw_error else None + + metadata = IndexMetadata( + index_id=cast(str, row.index_id), + resource_id=cast(str, row.resource_id), + indexed_at=_safe_fromisoformat(cast(str, row.indexed_at)), + file_count=len(file_list), + token_estimate=sum(f.token_count for f in file_list), + primary_language=cast(str, row.primary_language), + status=IndexStatus(cast(str, row.status)), + error_message=error_msg, + created_at=_safe_fromisoformat(cast(str, row.created_at)), + ) + + return RepoIndex(metadata=metadata, files=files) + finally: + session.close() + + +def load_index_status(session_factory: Any, resource_id: str) -> IndexMetadata | None: + """Load only the index metadata (no file records). + + Lightweight query for status display. + + Returns: + The :class:`IndexMetadata` if it exists, or ``None``. + """ + session = _session(session_factory) + try: + row = session.query(RepoIndexModel).filter_by(resource_id=resource_id).first() + if row is None: + return None + + raw_error = cast(str | None, row.error_message) + error_msg = raw_error if raw_error else None + + return IndexMetadata( + index_id=cast(str, row.index_id), + resource_id=cast(str, row.resource_id), + indexed_at=_safe_fromisoformat(cast(str, row.indexed_at)), + file_count=cast(int, row.file_count), + token_estimate=cast(int, row.token_estimate), + primary_language=cast(str, row.primary_language), + status=IndexStatus(cast(str, row.status)), + error_message=error_msg, + created_at=_safe_fromisoformat(cast(str, row.created_at)), + ) + finally: + session.close() diff --git a/src/cleveragents/application/services/repo_indexing_service.py b/src/cleveragents/application/services/repo_indexing_service.py new file mode 100644 index 000000000..f2dcb29a3 --- /dev/null +++ b/src/cleveragents/application/services/repo_indexing_service.py @@ -0,0 +1,499 @@ +"""Repository indexing service for CleverAgents. + +Provides indexing for 10K+ file projects with incremental refresh, +language detection, and policy enforcement (include/exclude globs, +size limits). Utilities in :mod:`~.repo_indexing_utils`; persistence +in :mod:`~.repo_indexing_persistence`. +""" + +from __future__ import annotations + +import functools +import logging +import re +import threading +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, cast + +from ulid import ULID + +from cleveragents.application.services.repo_indexing_persistence import ( + load_index, + load_index_status, + persist_index, + persist_status, +) +from cleveragents.application.services.repo_indexing_utils import ( + detect_language, + detect_primary_language, + estimate_tokens, + walk_and_index, +) +from cleveragents.domain.models.core.repo_index import ( + ULID_PATTERN, + FileRecord, + IndexMetadata, + IndexStatus, + RepoIndex, +) +from cleveragents.infrastructure.database.models import ( + IndexedFileModel, + RepoIndexModel, +) + +__all__ = [ + "RepoIndexingService", + "detect_language", + "estimate_tokens", +] + +logger = logging.getLogger(__name__) + + +def _serialize_on_resource[F: Callable[..., object]](method: F) -> F: + """Serialize concurrent calls for the same *resource_id*.""" + + @functools.wraps(method) + def _locked(self: Any, resource_id: str, *a: Any, **kw: Any) -> Any: + with self._resource_lock(resource_id): + return method(self, resource_id, *a, **kw) + + return cast(F, _locked) + + +class RepoIndexingService: + """Repository indexing service (session-factory pattern). + + Walks the file tree applying globs/size limits, hashes content, + detects languages, and persists results. Concurrent calls for the + same *resource_id* are serialized via ``_serialize_on_resource``. + + Each persistence helper opens its own database session, matching the + session-factory DI pattern used throughout the codebase. The + per-resource ``RLock`` serializes concurrent operations within a + single process; multi-process safety is not required (SQLite). + + INVARIANTS: (1) file_count == IndexedFileModel row count for + index_id; (2) token_estimate == sum(token_count); (3) + primary_language == mode language or "unknown"; (4) status "ready" + iff no error; (5) resource_id has at most one RepoIndexModel row. + """ + + def __init__(self, session_factory: Any) -> None: + """Initialise with *session_factory* (callable → SQLAlchemy Session).""" + self._session_factory = session_factory + self._resource_locks: dict[str, threading.RLock] = {} + self._locks_guard = threading.Lock() + self.cleanup_stale_indexing() + + def _resource_lock(self, resource_id: str) -> threading.RLock: + """Return a per-*resource_id* reentrant lock.""" + with self._locks_guard: + return self._resource_locks.setdefault(resource_id, threading.RLock()) + + @staticmethod + def _validate_resource_id(resource_id: str) -> None: + """Validate that *resource_id* is a non-empty ULID string.""" + if not resource_id: + raise ValueError("resource_id must be non-empty") + if not re.match(ULID_PATTERN, resource_id): + raise ValueError(f"resource_id must be a valid ULID, got: {resource_id!r}") + + # -- Public API ---------------------------------------------------------- + + @_serialize_on_resource + def index_resource( + self, + resource_id: str, + root_path: str | Path, + *, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), + max_file_size: int | None = None, + max_total_size: int | None = None, + ) -> RepoIndex: + """Index a resource by walking its file tree. + + Replaces any existing index (full re-index). For incremental + refresh use :meth:`refresh_index`. + + Raises: + FileNotFoundError: If *root_path* does not exist. + ValueError: If *resource_id* is empty or not a ULID. + """ + self._validate_resource_id(resource_id) + + root = Path(root_path) + if not root.exists(): + raise FileNotFoundError(f"Resource root path does not exist: {root}") + if not root.is_dir(): + raise ValueError(f"Resource root path is not a directory: {root}") + + index_id = str(ULID()) + + # Check whether a previous *good* (READY) index exists. If so, + # we do NOT overwrite it with an INDEXING placeholder — that + # would destroy the previous index if the walk fails (NEW-1). + # Stale INDEXING or ERROR rows from prior crashes are treated + # as "no good index" and re-initialised (F11/N1 fix). + existing_index = self.get_index(resource_id) + has_good_index = ( + existing_index is not None + and existing_index.metadata.status == IndexStatus.READY + ) + if not has_good_index: + persist_status( + self._session_factory, + index_id=index_id, + resource_id=resource_id, + status=IndexStatus.INDEXING, + ) + + logger.info( + "Indexing resource", + extra={ + "resource_id": resource_id, + "root_path": str(root), + "include_globs": include_globs, + "exclude_globs": exclude_globs, + }, + ) + + try: + file_records = walk_and_index( + root=root, + include_globs=include_globs, + exclude_globs=exclude_globs, + max_file_size=max_file_size, + max_total_size=max_total_size, + ) + except Exception as exc: + if not has_good_index: + try: + persist_status( + self._session_factory, + index_id=index_id, + resource_id=resource_id, + status=IndexStatus.ERROR, + error_message=f"{type(exc).__name__}: {str(exc)[:200]}", + ) + except Exception: + logger.warning( + "Failed to persist ERROR status after walk failure", + extra={"resource_id": resource_id}, + ) + else: + logger.warning( + "Indexing failed; preserving previous good index", + extra={ + "resource_id": resource_id, + "error": str(exc), + }, + ) + raise + + # Compute summary + file_count = len(file_records) + token_estimate = sum(fr.token_count for fr in file_records) + primary_language = detect_primary_language(file_records) + + now = datetime.now(tz=UTC) + + metadata = IndexMetadata( + index_id=index_id, + resource_id=resource_id, + indexed_at=now, + file_count=file_count, + token_estimate=token_estimate, + primary_language=primary_language, + status=IndexStatus.READY, + ) + + repo_index = RepoIndex(metadata=metadata, files=tuple(file_records)) + + # Persist (replace existing if any). + try: + persist_index(self._session_factory, repo_index) + except Exception as exc: + if not has_good_index: + try: + persist_status( + self._session_factory, + index_id=index_id, + resource_id=resource_id, + status=IndexStatus.ERROR, + error_message=( + f"persist failed: {type(exc).__name__}: {str(exc)[:200]}" + ), + ) + except Exception: + logger.warning( + "Failed to persist ERROR status after persist failure", + extra={"resource_id": resource_id}, + ) + raise + + logger.info( + "Indexing complete", + extra={ + "resource_id": resource_id, + "file_count": file_count, + "token_estimate": token_estimate, + "primary_language": primary_language, + }, + ) + + return repo_index + + @_serialize_on_resource + def refresh_index( + self, + resource_id: str, + root_path: str | Path, + *, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), + max_file_size: int | None = None, + max_total_size: int | None = None, + ) -> RepoIndex: + """Incrementally refresh an existing index. + + Only files whose content hash has changed (or that are new/deleted) + are re-processed. If no prior index exists, performs a full index. + Note: ``last_modified`` for unchanged files retains the value from + the previous index (not re-read from disk) to avoid stat overhead. + + Args: + resource_id: ULID of the resource. + root_path: Filesystem path to the resource root. + include_globs: File path globs to include (empty = all). + exclude_globs: File path globs to exclude. + max_file_size: Max individual file size in bytes. + max_total_size: Max total size in bytes. + + Returns: + The refreshed :class:`RepoIndex`. + + Raises: + FileNotFoundError: If *root_path* does not exist. + ValueError: If *resource_id* is empty. + """ + self._validate_resource_id(resource_id) + + existing = self.get_index(resource_id) + if existing is None: + return self.index_resource( + resource_id, + root_path, + include_globs=include_globs, + exclude_globs=exclude_globs, + max_file_size=max_file_size, + max_total_size=max_total_size, + ) + + # Same validation as index_resource (DRY extraction blocked by 500-line limit). + root = Path(root_path) + if not root.exists(): + raise FileNotFoundError(f"Resource root path does not exist: {root}") + if not root.is_dir(): + raise ValueError(f"Resource root path is not a directory: {root}") + + logger.info( + "Refreshing index incrementally", + extra={"resource_id": resource_id}, + ) + + try: + current_files = walk_and_index( + root=root, + include_globs=include_globs, + exclude_globs=exclude_globs, + max_file_size=max_file_size, + max_total_size=max_total_size, + ) + except Exception as exc: + logger.warning( + "Incremental refresh failed; preserving previous good index", + extra={ + "resource_id": resource_id, + "error": str(exc), + }, + ) + raise + + # Merge: reuse existing records for unchanged files. + existing_by_path: dict[str, FileRecord] = {fr.path: fr for fr in existing.files} + current_paths: set[str] = set() + merged: list[FileRecord] = [] + changed_count = 0 + for new_fr in current_files: + current_paths.add(new_fr.path) + old_record = existing_by_path.get(new_fr.path) + if ( + old_record is not None + and old_record.content_hash == new_fr.content_hash + ): + merged.append(old_record) + else: + merged.append(new_fr) + changed_count += 1 + + # Count files present in old index but absent on disk (N12 fix). + deleted_count = sum(1 for p in existing_by_path if p not in current_paths) + + file_count = len(merged) + token_estimate = sum(fr.token_count for fr in merged) + primary_language = detect_primary_language(merged) + + now = datetime.now(tz=UTC) + + metadata = IndexMetadata( + index_id=existing.metadata.index_id, + resource_id=resource_id, + indexed_at=now, + file_count=file_count, + token_estimate=token_estimate, + primary_language=primary_language, + status=IndexStatus.READY, + ) + + repo_index = RepoIndex(metadata=metadata, files=tuple(merged)) + + # Persist with error handling (G2 fix). + try: + persist_index(self._session_factory, repo_index) + except Exception as exc: + logger.warning( + "Failed to persist refreshed index; preserving previous", + extra={ + "resource_id": resource_id, + "error": str(exc), + }, + ) + raise + + logger.info( + "Incremental refresh complete", + extra={ + "resource_id": resource_id, + "file_count": file_count, + "changed_count": changed_count, + "deleted_count": deleted_count, + }, + ) + + return repo_index + + def get_index(self, resource_id: str) -> RepoIndex | None: + """Retrieve the persisted index for a resource. + + Args: + resource_id: ULID of the resource. + + Returns: + The :class:`RepoIndex` if it exists, or ``None``. + + Raises: + ValueError: If *resource_id* is empty. + """ + self._validate_resource_id(resource_id) + return load_index(self._session_factory, resource_id) + + def get_index_status(self, resource_id: str) -> IndexMetadata | None: + """Retrieve only the index metadata (no file records). + + Lightweight query for status display (e.g., ``agents project show``). + + Args: + resource_id: ULID of the resource. + + Returns: + The :class:`IndexMetadata` if it exists, or ``None``. + + Raises: + ValueError: If *resource_id* is empty. + """ + self._validate_resource_id(resource_id) + return load_index_status(self._session_factory, resource_id) + + @_serialize_on_resource + def remove_index(self, resource_id: str) -> bool: + """Remove the index for a resource. + + Args: + resource_id: ULID of the resource. + + Returns: + ``True`` if an index was removed, ``False`` if none existed. + + Raises: + ValueError: If *resource_id* is empty. + """ + self._validate_resource_id(resource_id) + session = self._session_factory() + try: + row = ( + session.query(RepoIndexModel).filter_by(resource_id=resource_id).first() + ) + if row is None: + return False + + session.query(IndexedFileModel).filter_by(index_id=row.index_id).delete() + session.delete(row) + session.commit() + + # Evict per-resource lock — no further ops expected. + with self._locks_guard: + self._resource_locks.pop(resource_id, None) + logger.info( + "Removed index", + extra={ + "resource_id": resource_id, + "index_id": row.index_id, + }, + ) + return True + except Exception: + session.rollback() + raise + finally: + session.close() + + def cleanup_stale_indexing(self) -> int: + """Remove orphan ``INDEXING`` rows left by crashed processes. + + Should be called at application startup. Any row with + ``status='indexing'`` is a remnant of an incomplete index + operation (the process crashed before writing ``READY`` or + ``ERROR``). + + Returns: + Number of stale rows removed. + """ + session = self._session_factory() + try: + stale_rows = ( + session.query(RepoIndexModel) + .filter_by(status=IndexStatus.INDEXING.value) + .all() + ) + count = len(stale_rows) + for row in stale_rows: + session.query(IndexedFileModel).filter_by( + index_id=row.index_id + ).delete() + session.delete(row) + if count: + session.commit() + logger.info( + "Cleaned up stale INDEXING rows", + extra={"count": count}, + ) + return count + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/src/cleveragents/application/services/repo_indexing_utils.py b/src/cleveragents/application/services/repo_indexing_utils.py new file mode 100644 index 000000000..694aa1b75 --- /dev/null +++ b/src/cleveragents/application/services/repo_indexing_utils.py @@ -0,0 +1,382 @@ +"""Pure utility functions for repository indexing. + +Extracted from :mod:`~cleveragents.application.services.repo_indexing_service` +to keep the service module under the 500-line limit (CONTRIBUTING.md line 396). + +All functions in this module are stateless and do not depend on SQLAlchemy, +the DI container, or any other infrastructure. They are safe to call from +tests and benchmarks without constructing a full service instance. +""" + +from __future__ import annotations + +import fnmatch +import hashlib +import logging +import os +import stat as stat_mod +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath + +from cleveragents.domain.models.core.repo_index import FileRecord + +logger = logging.getLogger(__name__) + +__all__ = [ + "detect_language", + "detect_primary_language", + "estimate_tokens", + "matches_policy", + "walk_and_index", +] + +# --------------------------------------------------------------------------- +# Language detection (extension-based) +# --------------------------------------------------------------------------- + +# Spec reference: context.uko.default-analyzers (line 28694) +# ["python", "typescript", "rust", "java", "markdown", "json-schema"] +_EXTENSION_LANGUAGE_MAP: dict[str, str] = { + ".py": "python", + ".pyi": "python", + ".pyx": "python", + ".ts": "typescript", + ".tsx": "typescript", + ".js": "javascript", + ".jsx": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".rs": "rust", + ".java": "java", + ".kt": "kotlin", + ".kts": "kotlin", + ".go": "go", + ".c": "c", + ".h": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".hpp": "cpp", + ".cs": "csharp", + ".rb": "ruby", + ".php": "php", + ".swift": "swift", + ".scala": "scala", + ".r": "r", + ".md": "markdown", + ".mdx": "markdown", + ".rst": "restructuredtext", + ".json": "json", + ".yaml": "yaml", + ".yml": "yaml", + ".toml": "toml", + ".xml": "xml", + ".html": "html", + ".htm": "html", + ".css": "css", + ".scss": "css", + ".sql": "sql", + ".sh": "shell", + ".bash": "shell", + ".zsh": "shell", + ".ps1": "powershell", + ".dockerfile": "dockerfile", + ".tf": "terraform", + ".lua": "lua", + ".zig": "zig", + ".nim": "nim", + ".ex": "elixir", + ".exs": "elixir", + ".erl": "erlang", + ".hs": "haskell", + ".ml": "ocaml", + ".mli": "ocaml", + ".clj": "clojure", + ".dart": "dart", + ".v": "v", # V language; also Verilog — ambiguous by design + ".jl": "julia", +} + +# Token estimation: ~4 characters per token (conservative average) +_CHARS_PER_TOKEN = 4 + + +def detect_language(path: str) -> str: + """Detect programming language from file extension. + + Args: + path: File path (only the extension is used). + + Returns: + Lowercase language name, or ``"unknown"`` if unrecognised. + """ + ext = Path(path).suffix.lower() + # Special case: Dockerfile without extension + basename = Path(path).name.lower() + if basename == "dockerfile" or basename.startswith("dockerfile."): + return "dockerfile" + if basename in ("makefile", "gnumakefile"): + return "makefile" + return _EXTENSION_LANGUAGE_MAP.get(ext, "unknown") + + +def estimate_tokens(content_bytes: int) -> int: + """Estimate token count from byte size. + + Uses a conservative 4 characters per token ratio. + + Args: + content_bytes: File size in bytes. + + Returns: + Estimated token count (non-negative). + """ + return max(0, content_bytes // _CHARS_PER_TOKEN) + + +# --------------------------------------------------------------------------- +# File hashing +# --------------------------------------------------------------------------- + + +def read_and_hash(path: Path) -> tuple[str, int]: + """Read a file, return ``(sha256_hex, byte_length)``. + + Reads the file exactly once so that hash and size are always + consistent (no TOCTOU window). + """ + h = hashlib.sha256() + length = 0 + with open(path, "rb") as f: + while True: + chunk = f.read(65536) + if not chunk: + break + h.update(chunk) + length += len(chunk) + return h.hexdigest(), length + + +# --------------------------------------------------------------------------- +# Glob matching +# --------------------------------------------------------------------------- + + +def _glob_match(rel_path: str, pattern: str) -> bool: + """Match *rel_path* against a glob *pattern*. + + Uses :meth:`pathlib.PurePosixPath.match` which supports ``**`` + for recursive directory matching (N5 fix). Falls back to + :func:`fnmatch.fnmatch` for simple patterns without ``/`` so that + ``*.py`` still matches ``foo.py`` without requiring ``**/*.py``. + """ + if "**" in pattern or "/" in pattern: + return PurePosixPath(rel_path).match(pattern) + return fnmatch.fnmatch(rel_path, pattern) + + +def matches_policy( + rel_path: str, + include_globs: tuple[str, ...], + exclude_globs: tuple[str, ...], +) -> bool: + """Check if a file path matches the include/exclude policy. + + - If *include_globs* is non-empty, path must match at least one. + - If *exclude_globs* is non-empty, path must not match any. + - Exclusions take precedence over inclusions. + """ + # Exclude takes precedence + for pattern in exclude_globs: + if _glob_match(rel_path, pattern): + return False + + # If no include globs, everything is included + if not include_globs: + return True + + # Must match at least one include glob + return any(_glob_match(rel_path, pat) for pat in include_globs) + + +# --------------------------------------------------------------------------- +# Primary language detection +# --------------------------------------------------------------------------- + + +def detect_primary_language(records: list[FileRecord]) -> str: + """Determine the primary language from file records. + + Languages are weighted by token count so that a few large source + files outweigh many small config files (G6 fix). Returns + ``"unknown"`` if all files are unrecognised or have zero tokens. + """ + lang_tokens: Counter[str] = Counter() + for fr in records: + if fr.language != "unknown": + lang_tokens[fr.language] += fr.token_count + + if not lang_tokens: + return "unknown" + + return lang_tokens.most_common(1)[0][0] + + +# --------------------------------------------------------------------------- +# Filesystem walk +# --------------------------------------------------------------------------- + + +def walk_and_index( + root: Path, + *, + include_globs: tuple[str, ...], + exclude_globs: tuple[str, ...], + max_file_size: int | None, + max_total_size: int | None, + max_file_count: int | None = None, +) -> list[FileRecord]: + """Walk a directory tree and build file records. + + Applies include/exclude glob filtering and size limits. + + Files are collected and sorted by relative path before applying the + ``max_total_size`` and ``max_file_count`` cutoffs, ensuring + deterministic results regardless of filesystem walk order. + + ``max_file_count`` is an internal parameter used only by benchmarks + and tests — not exposed in the public ``RepoIndexingService`` API. + + The ``max_total_size`` cutoff uses Phase-1 ``stat`` sizes for the + budget check; actual byte count (from ``read_and_hash``) may differ + slightly if files are modified between stat and read. + """ + # Canonicalize root so that relative_to() always works and + # "../" traversal cannot escape the intended scope (G5 fix). + root = root.resolve() + + # Phase 1: Collect all candidate files (path, stat) sorted by rel_path + candidates: list[tuple[str, Path, os.stat_result]] = [] + + for dirpath, dirnames, filenames in os.walk(root): + # Skip hidden directories (startswith(".") covers .git, .venv, + # .tox, .mypy_cache, etc.) and common non-source directories. + dirnames[:] = sorted( + d + for d in dirnames + if not d.startswith(".") + and d not in {"__pycache__", "node_modules", "venv"} + ) + + for filename in sorted(filenames): + if filename.startswith("."): + continue + + full_path = Path(dirpath) / filename + + # Skip symlinks to avoid reading content outside the + # repository root. + if full_path.is_symlink(): + continue + + try: + rel_path = str(full_path.relative_to(root)) + except ValueError: + continue + + # Skip paths that exceed FileRecord.path max_length (1024) + # to avoid a Pydantic ValidationError that would crash the + # entire index. + if len(rel_path) > 1024: + logger.debug( + "Skipping file with path > 1024 chars", + extra={"path": rel_path[:200]}, + ) + continue + + # Apply include/exclude globs + if not matches_policy(rel_path, include_globs, exclude_globs): + logger.debug("Excluded by glob policy", extra={"path": rel_path}) + continue + + # Size check + try: + stat = full_path.stat() + except OSError: + continue + + # Only index regular files — skip symlinks, directories, + # FIFOs, sockets, and other special entries (S3 fix). + if not stat_mod.S_ISREG(stat.st_mode): + continue + + size_bytes = stat.st_size + + if max_file_size is not None and size_bytes > max_file_size: + logger.debug( + "Skipping file exceeding max_file_size", + extra={ + "path": rel_path, + "size": size_bytes, + "limit": max_file_size, + }, + ) + continue + + candidates.append((rel_path, full_path, stat)) + + # Sort by relative path for deterministic max_total_size truncation + candidates.sort(key=lambda c: c[0]) + + # Phase 2: Build records, applying max_total_size cutoff + records: list[FileRecord] = [] + total_bytes = 0 + + for rel_path, full_path, stat in candidates: + if max_file_count is not None and len(records) >= max_file_count: + logger.debug( + "Reached max_file_count limit, stopping", + extra={"file_count": len(records), "limit": max_file_count}, + ) + break + + size_bytes = stat.st_size + + if max_total_size is not None and total_bytes + size_bytes > max_total_size: + logger.debug( + "Reached max_total_size limit, stopping", + extra={"total_bytes": total_bytes, "limit": max_total_size}, + ) + break + + # Re-check symlink before reading to narrow TOCTOU window. + if full_path.is_symlink(): + continue + # Single read: hash content and measure size atomically so + # hash and size_bytes are always consistent (N2/N3 fix). + # Use Phase-1 stat mtime to avoid a second stat() call. + try: + content_hash, actual_size = read_and_hash(full_path) + mtime = datetime.fromtimestamp(stat.st_mtime, tz=UTC) + except OSError: + continue + + size_bytes = actual_size + total_bytes += size_bytes + + lang = detect_language(rel_path) + tokens = estimate_tokens(size_bytes) + + records.append( + FileRecord( + path=rel_path, + content_hash=content_hash, + token_count=tokens, + language=lang, + size_bytes=size_bytes, + last_modified=mtime, + ) + ) + + return records diff --git a/src/cleveragents/domain/models/acms/_postgresql_helpers.py b/src/cleveragents/domain/models/acms/_postgresql_helpers.py index f664f4139..cf5fdf7e6 100644 --- a/src/cleveragents/domain/models/acms/_postgresql_helpers.py +++ b/src/cleveragents/domain/models/acms/_postgresql_helpers.py @@ -9,38 +9,75 @@ Not part of the public API -- do not import directly. from __future__ import annotations +import logging import re +from cleveragents.domain.models.acms._sql_string_aware import ( + find_unquoted_semicolon as find_unquoted_semicolon, +) +from cleveragents.domain.models.acms._sql_string_aware import ( + strip_sql_comments as strip_sql_comments, +) from cleveragents.domain.models.acms.analyzers import UKOTriple +logger = logging.getLogger(__name__) + __all__: list[str] = [] # --------------------------------------------------------------------------- # Regex patterns # --------------------------------------------------------------------------- +# Identifier fragment: matches "quoted-ident" or unquoted_ident (incl. $). +# Each _IDENT usage creates TWO groups: (quoted, unquoted). +# Quoted branch allows escaped double-quotes ("") inside identifiers. +_IDENT = r'(?:"((?:[^"]|"")+)"|([\w$]+))' + +# Dollar-quoting: $$...$$ or $tag$...$tag$ (PostgreSQL string literals). +_DOLLAR_RE = re.compile(r"\$(\w*)\$") + + +def ident_pair(match: re.Match[str], g1: int, g2: int) -> str | None: + """Return the identifier captured by an ``_IDENT`` group pair. + + Each ``_IDENT`` in a regex produces *two* groups — one for the + quoted branch and one for the unquoted branch. Exactly one is + non-``None``. This helper collapses them into a single value and + un-escapes doubled double-quotes (``""`` → ``"``). + """ + quoted = match.group(g1) + if quoted is not None: + return quoted.replace('""', '"') + return match.group(g2) + + CREATE_TABLE_RE = re.compile( - r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" - r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(", + r"CREATE\s+" + r"(?:(?:GLOBAL\s+|LOCAL\s+)?(?:TEMP(?:ORARY)?\s+)|UNLOGGED\s+)?" + r"TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" + rf"(?:{_IDENT}\.)?{_IDENT}\s*\(", re.IGNORECASE, ) CREATE_VIEW_RE = re.compile( - r"CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+" - r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s+AS\b", + r"CREATE\s+" + r"(?:OR\s+REPLACE\s+)?" + r"(?:(?:TEMP(?:ORARY)?\s+)|(?:MATERIALIZED\s+))?" + r"VIEW\s+" + rf"(?:{_IDENT}\.)?{_IDENT}\s+AS\b", re.IGNORECASE, ) CREATE_SCHEMA_RE = re.compile( - r"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?", + rf"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}", re.IGNORECASE, ) COLUMN_DEF_RE = re.compile( - r"^\s*\"?(\w+)\"?\s+" + r'^\s*(?:"([^"]+)"|([\w$]+))\s+' r"((?:CHARACTER\s+VARYING|DOUBLE\s+PRECISION" r"|TIME(?:STAMP)?\s+WITH(?:OUT)?\s+TIME\s+ZONE" - r"|\w+)" + r"|[\w$]+)" r"(?:\s*\([^)]*\))?(?:\s*\[\s*\])?)", re.IGNORECASE, ) @@ -51,7 +88,7 @@ PRIMARY_KEY_INLINE_RE = re.compile(r"\bPRIMARY\s+KEY\b", re.IGNORECASE) FOREIGN_KEY_RE = re.compile( r"FOREIGN\s+KEY\s*\(\s*([^)]+)\)\s*" - r"REFERENCES\s+(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(\s*([^)]+)\)", + rf"REFERENCES\s+(?:{_IDENT}\.)?{_IDENT}\s*\(\s*([^)]+)\)", re.IGNORECASE, ) @@ -132,56 +169,63 @@ def schema_uri(resource_uri: str, schema_name: str) -> str: return f"uko://data/schema/{safe_name(resource_uri)}/{safe_name(schema_name)}" -# --------------------------------------------------------------------------- -# SQL comment stripping -# --------------------------------------------------------------------------- - -_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) -_LINE_COMMENT_RE = re.compile(r"--[^\n]*") - - -def strip_sql_comments(content: str) -> str: - """Remove SQL block (``/* */``) and line (``--``) comments.""" - content = _BLOCK_COMMENT_RE.sub("", content) - return _LINE_COMMENT_RE.sub("", content) - - # --------------------------------------------------------------------------- # Parenthesis / body extraction # --------------------------------------------------------------------------- +def _skip_quoted(text: str, i: int) -> int: + """Advance past a quoted region starting at *i*. + + Handles single-quoted strings (``''`` escape), double-quoted + identifiers (``""`` escape), and dollar-quoted strings. + Returns the index just past the closing delimiter, or ``len(text)`` + if the closing delimiter is never found. + """ + ch = text[i] + n = len(text) + if ch in ("'", '"'): + j = i + 1 + while j < n: + if text[j] == ch: + if j + 1 < n and text[j + 1] == ch: + j += 2 + else: + return j + 1 + else: + j += 1 + return n + if ch == "$": + m = _DOLLAR_RE.match(text, i) + if m: + tag: str = m.group(0) + end = text.find(tag, m.end()) + return (end + len(tag)) if end != -1 else n + return i + 1 + + def extract_body(content: str, paren_start: int) -> str: """Return text between balanced parentheses starting at *paren_start*. - Respects single-quoted SQL string literals so that parentheses - inside strings (e.g. ``DEFAULT 'func(x)'``) do not affect the - depth counter. PostgreSQL ``''`` escape is handled by skipping - two consecutive single-quote characters. + Respects single-quoted strings, double-quoted identifiers, and + dollar-quoted strings so that parentheses inside them do not + affect the depth counter. """ if paren_start >= len(content) or content[paren_start] != "(": return "" depth = 0 - in_string = False i = paren_start while i < len(content): ch = content[i] - if in_string: - if ch == "'": - # '' is an escaped quote inside a string literal. - if i + 1 < len(content) and content[i + 1] == "'": - i += 2 - continue - in_string = False - else: - if ch == "'": - in_string = True - elif ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth == 0: - return content[paren_start + 1 : i] + if ch in ("'", '"') or (ch == "$" and _DOLLAR_RE.match(content, i)): + i = _skip_quoted(content, i) + continue + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return content[paren_start + 1 : i] i += 1 return "" @@ -189,40 +233,32 @@ def extract_body(content: str, paren_start: int) -> str: def split_entries(body: str) -> list[str]: """Split a table body into entries by commas at depth 0. - Respects single-quoted SQL string literals so that commas and - parentheses inside strings are ignored. + Respects single-quoted strings, double-quoted identifiers, and + dollar-quoted strings so that commas and parentheses inside them + are ignored. """ entries: list[str] = [] depth = 0 - in_string = False current: list[str] = [] i = 0 while i < len(body): ch = body[i] - if in_string: + if ch in ("'", '"') or (ch == "$" and _DOLLAR_RE.match(body, i)): + start = i + i = _skip_quoted(body, i) + current.append(body[start:i]) + continue + if ch == "(": + depth += 1 current.append(ch) - if ch == "'": - # '' escape inside a string literal. - if i + 1 < len(body) and body[i + 1] == "'": - current.append(body[i + 1]) - i += 2 - continue - in_string = False + elif ch == ")": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + entries.append("".join(current)) + current = [] else: - if ch == "'": - in_string = True - current.append(ch) - elif ch == "(": - depth += 1 - current.append(ch) - elif ch == ")": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - entries.append("".join(current)) - current = [] - else: - current.append(ch) + current.append(ch) i += 1 tail = "".join(current).strip() if tail: @@ -320,10 +356,10 @@ def extract_fk_triples( source_cols = [ c.strip().strip('"') for c in fk_match.group(1).split(",") if c.strip() ] - ref_schema = fk_match.group(2) or schema_name - ref_table = fk_match.group(3) + ref_schema = ident_pair(fk_match, 2, 3) or schema_name + ref_table = ident_pair(fk_match, 4, 5) or "" target_cols = [ - c.strip().strip('"') for c in fk_match.group(4).split(",") if c.strip() + c.strip().strip('"') for c in fk_match.group(6).split(",") if c.strip() ] first_col = source_cols[0] if source_cols else "_unknown_" @@ -344,6 +380,14 @@ def extract_fk_triples( ) ) + if len(source_cols) != len(target_cols): + logger.warning( + "FK column count mismatch: %d source vs %d target columns; " + "pairing only the shorter list (A8 fix)", + len(source_cols), + len(target_cols), + ) + for src_col, tgt_col in zip(source_cols, target_cols, strict=False): source_col_uri = column_uri(resource_uri, table_name, src_col, schema_name) target_col_uri = column_uri(resource_uri, ref_table, tgt_col, ref_schema) @@ -416,13 +460,19 @@ def parse_table_body( # Column definition col_match = COLUMN_DEF_RE.match(stripped) if col_match: - col_name = col_match.group(1) - col_type = col_match.group(2).strip() + col_name = col_match.group(1) or col_match.group(2) + col_type = col_match.group(3).strip() remainder = stripped[col_match.end() :] is_not_null = bool(NOT_NULL_RE.search(remainder)) is_pk_inline = bool(PRIMARY_KEY_INLINE_RE.search(remainder)) is_pk = is_pk_inline or col_name.lower() in pk_columns + # SERIAL/BIGSERIAL/SMALLSERIAL are implicitly NOT NULL (A7 fix). + is_serial = col_type.upper() in { + "SERIAL", + "BIGSERIAL", + "SMALLSERIAL", + } triples.extend( emit_column_triples( @@ -432,7 +482,7 @@ def parse_table_body( col_name, col_type, schema_name=schema_name, - is_nullable=not (is_not_null or is_pk), + is_nullable=not (is_not_null or is_pk or is_serial), is_primary_key=is_pk, ) ) diff --git a/src/cleveragents/domain/models/acms/_sql_string_aware.py b/src/cleveragents/domain/models/acms/_sql_string_aware.py new file mode 100644 index 000000000..dc774f12f --- /dev/null +++ b/src/cleveragents/domain/models/acms/_sql_string_aware.py @@ -0,0 +1,134 @@ +"""String-literal-aware SQL scanning utilities. + +Single-pass state-machine scanner for stripping SQL comments and +finding unquoted semicolons while respecting single-quoted strings, +dollar-quoted strings, and nested block comments. + +Internal module — do not import directly. +""" + +from __future__ import annotations + +import re + +__all__: list[str] = [] + +_DOLLAR_TAG_RE = re.compile(r"\$([A-Za-z_][\w]*)?(\$)") + + +def strip_sql_comments(content: str) -> str: + """Remove SQL comments while respecting string literals. + + Handles: + - Single-quoted strings with ``''`` escape (``'it''s'``). + - Dollar-quoted strings (``$$body$$``, ``$tag$body$tag$``). + - Nested block comments (``/* outer /* inner */ still comment */``). + - Line comments (``-- ...``). + """ + out: list[str] = [] + i = 0 + n = len(content) + + while i < n: + c = content[i] + + # --- single-quoted string --- + if c == "'": + j = i + 1 + while j < n: + if content[j] == "'" and j + 1 < n and content[j + 1] == "'": + j += 2 # escaped '' + elif content[j] == "'": + j += 1 + break + else: + j += 1 + out.append(content[i:j]) + i = j + continue + + # --- dollar-quoted string --- + if c == "$": + m = _DOLLAR_TAG_RE.match(content, i) + if m: + tag = content[i : m.end()] # e.g. $$ or $tag$ + end = content.find(tag, m.end()) + if end != -1: + out.append(content[i : end + len(tag)]) + i = end + len(tag) + else: + out.append(content[i:]) + i = n + continue + + # --- block comment (supports nesting) --- + if c == "/" and i + 1 < n and content[i + 1] == "*": + depth = 1 + j = i + 2 + while j < n and depth > 0: + if content[j] == "/" and j + 1 < n and content[j + 1] == "*": + depth += 1 + j += 2 + elif content[j] == "*" and j + 1 < n and content[j + 1] == "/": + depth -= 1 + j += 2 + else: + j += 1 + i = j + continue + + # --- line comment --- + if c == "-" and i + 1 < n and content[i + 1] == "-": + j = i + 2 + while j < n and content[j] != "\n": + j += 1 + i = j + continue + + out.append(c) + i += 1 + + return "".join(out) + + +def find_unquoted_semicolon(content: str, start: int = 0) -> int: + """Find the first ``;`` not inside a string literal. + + Returns the index of the semicolon, or ``-1`` if not found. + Respects single-quoted strings (with ``''`` escapes) and + dollar-quoted strings. + """ + i = start + n = len(content) + + while i < n: + c = content[i] + + if c == ";": + return i + + # skip single-quoted string + if c == "'": + i += 1 + while i < n: + if content[i] == "'" and i + 1 < n and content[i + 1] == "'": + i += 2 + elif content[i] == "'": + i += 1 + break + else: + i += 1 + continue + + # skip dollar-quoted string + if c == "$": + m = _DOLLAR_TAG_RE.match(content, i) + if m: + tag = content[i : m.end()] + end = content.find(tag, m.end()) + i = (end + len(tag)) if end != -1 else n + continue + + i += 1 + + return -1 diff --git a/src/cleveragents/domain/models/acms/analyzers.py b/src/cleveragents/domain/models/acms/analyzers.py index 9d2b2a66d..1a12df2df 100644 --- a/src/cleveragents/domain/models/acms/analyzers.py +++ b/src/cleveragents/domain/models/acms/analyzers.py @@ -23,6 +23,7 @@ Language-Specific Analyzers, and configuration key from __future__ import annotations import logging +import re from typing import Protocol, runtime_checkable from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -87,6 +88,25 @@ class UKOTriple(BaseModel): return value +# --------------------------------------------------------------------------- +# URI segment sanitisation (shared by all analyzers) +# --------------------------------------------------------------------------- + +_SAFE_URI_RE = re.compile(r"[^a-zA-Z0-9_.-]") + + +def safe_uri_segment(text: str) -> str: + """Sanitise *text* for use in a UKO URI path segment. + + Replaces non-alphanumeric characters (except ``_``, ``.``, ``-``) + with underscores, strips leading/trailing underscores, and + truncates to 120 characters. Returns ``"_unknown_"`` if the + result is empty. + """ + result = _SAFE_URI_RE.sub("_", text).strip("_")[:120] + return result if result else "_unknown_" + + # --------------------------------------------------------------------------- # AnalyzerProtocol # --------------------------------------------------------------------------- diff --git a/src/cleveragents/domain/models/acms/docker_compose_analyzer.py b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py index 9dda92e40..1cb00f01d 100644 --- a/src/cleveragents/domain/models/acms/docker_compose_analyzer.py +++ b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py @@ -27,11 +27,14 @@ Based on ``docs/specification.md`` §42285-42331 — DockerComposeAnalyzer. from __future__ import annotations import logging -import re import yaml -from cleveragents.domain.models.acms.analyzers import AnalyzerProtocol, UKOTriple +from cleveragents.domain.models.acms.analyzers import ( + AnalyzerProtocol, + UKOTriple, + safe_uri_segment, +) __all__ = ["DockerComposeAnalyzer"] @@ -45,17 +48,7 @@ _MAX_COMPOSE_BYTES = 1_048_576 # 1 MiB # URI helpers # --------------------------------------------------------------------------- -_SAFE_RE = re.compile(r"[^a-zA-Z0-9_.-]") - - -def _safe(text: str) -> str: - """Sanitise text for use in a URI path segment. - - Truncates to 120 characters. Callers should be aware that very - long inputs sharing a common prefix may collide after truncation. - """ - result = _SAFE_RE.sub("_", text).strip("_")[:120] - return result if result else "_unknown_" +_safe = safe_uri_segment # B4 fix: shared implementation def _deployment_uri(resource_uri: str) -> str: @@ -145,7 +138,7 @@ class DockerComposeAnalyzer: # Size guard: reject oversized inputs to mitigate billion-laughs # alias-expansion attacks (quadratic memory via nested YAML # anchors/aliases). 1 MiB is generous for any Compose file. - if len(content) > _MAX_COMPOSE_BYTES: + if len(content.encode("utf-8")) > _MAX_COMPOSE_BYTES: logger.warning( "DockerComposeAnalyzer: content exceeds %d byte limit; skipping '%s'", _MAX_COMPOSE_BYTES, @@ -153,6 +146,9 @@ class DockerComposeAnalyzer: ) return [] + # yaml.safe_load disables custom constructors, limiting alias + # expansion impact. The _MAX_COMPOSE_BYTES check above provides + # a further guard against decompression-bomb-style YAML aliases. try: data = yaml.safe_load(content) except yaml.YAMLError: @@ -198,58 +194,74 @@ class DockerComposeAnalyzer: return triples for service_name, service_def in services.items(): - svc_uri = _service_uri(resource_uri, str(service_name)) + try: + svc_uri = _service_uri(resource_uri, str(service_name)) - # Service declaration - triples.append( - UKOTriple( - subject_uri=svc_uri, - predicate="rdf:type", - object_uri="uko-infra:Service", + # Service declaration + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="rdf:type", + object_uri="uko-infra:Service", + ) ) - ) - triples.append( - UKOTriple( - subject_uri=svc_uri, - predicate="rdfs:label", - object_value=str(service_name), + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="rdfs:label", + object_value=str(service_name), + ) ) - ) - # Containment: deployment unit -> service - triples.append( - UKOTriple( - subject_uri=deploy_uri, - predicate="uko:contains", - object_uri=svc_uri, + # Containment: deployment unit -> service + triples.append( + UKOTriple( + subject_uri=deploy_uri, + predicate="uko:contains", + object_uri=svc_uri, + ) ) - ) - # Guard against null service definitions (e.g. `web:` with - # no body). - if not isinstance(service_def, dict): + # Guard against null service definitions (e.g. `web:` with + # no body). + if not isinstance(service_def, dict): + continue + + # Ports + triples.extend( + self._extract_ports( + service_def, resource_uri, service_name, svc_uri + ) + ) + + # Environment variables + triples.extend( + self._extract_environment( + service_def, resource_uri, service_name, svc_uri + ) + ) + + # Dependencies + triples.extend( + self._extract_depends_on(service_def, resource_uri, svc_uri) + ) + + # Volumes + triples.extend( + self._extract_volumes( + service_def, resource_uri, service_name, svc_uri + ) + ) + except Exception: + logger.warning( + "DockerComposeAnalyzer: error processing service '%s' " + "in '%s'; skipping", + service_name, + resource_uri, + exc_info=True, + ) continue - # Ports - triples.extend( - self._extract_ports(service_def, resource_uri, service_name, svc_uri) - ) - - # Environment variables - triples.extend( - self._extract_environment( - service_def, resource_uri, service_name, svc_uri - ) - ) - - # Dependencies - triples.extend(self._extract_depends_on(service_def, resource_uri, svc_uri)) - - # Volumes - triples.extend( - self._extract_volumes(service_def, resource_uri, service_name, svc_uri) - ) - return triples # -- Internal extraction helpers ------------------------------------------ @@ -272,8 +284,10 @@ class DockerComposeAnalyzer: # human-readable string instead of Python dict repr. if isinstance(port_entry, dict): target = port_entry.get("target", "") - published = port_entry.get("published", "") - port_str = f"{published}:{target}" if published else str(target) + published = port_entry.get("published") + port_str = ( + f"{published}:{target}" if published is not None else str(target) + ) else: port_str = str(port_entry) p_uri = _port_uri(resource_uri, service_name, port_str) diff --git a/src/cleveragents/domain/models/acms/markdown_analyzer.py b/src/cleveragents/domain/models/acms/markdown_analyzer.py index e4286ffc0..9ef675c34 100644 --- a/src/cleveragents/domain/models/acms/markdown_analyzer.py +++ b/src/cleveragents/domain/models/acms/markdown_analyzer.py @@ -26,7 +26,11 @@ from __future__ import annotations import logging import re -from cleveragents.domain.models.acms.analyzers import AnalyzerProtocol, UKOTriple +from cleveragents.domain.models.acms.analyzers import ( + AnalyzerProtocol, + UKOTriple, + safe_uri_segment, +) logger = logging.getLogger(__name__) @@ -39,16 +43,14 @@ _FENCED_OPEN_RE = re.compile(r"^```(\w*)") _FENCED_CLOSE_RE = re.compile(r"^```\s*$") _LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +# Maximum content size to parse (1 MiB, same cap as DockerComposeAnalyzer). +_MAX_MARKDOWN_BYTES = 1024 * 1024 + # --------------------------------------------------------------------------- # URI helpers # --------------------------------------------------------------------------- -_SAFE_RE = re.compile(r"[^a-zA-Z0-9_.-]") - - -def _safe(text: str) -> str: - """Sanitise text for use in a URI path segment.""" - return _SAFE_RE.sub("_", text).strip("_")[:120] +_safe = safe_uri_segment # B4 fix: shared implementation def _doc_uri(resource_uri: str) -> str: @@ -88,7 +90,7 @@ class MarkdownAnalyzer: @property def supported_extensions(self) -> frozenset[str]: """File extensions handled by this analyzer.""" - return frozenset({".md", ".markdown"}) + return frozenset({".md", ".markdown", ".mdx"}) @property def domain(self) -> str: @@ -108,10 +110,16 @@ class MarkdownAnalyzer: Raises: ValueError: If *content* or *resource_uri* is empty. """ - if not content: + if not content or not content.strip(): raise ValueError("content must not be empty.") if not resource_uri: raise ValueError("resource_uri must not be empty.") + if len(content.encode("utf-8", errors="replace")) > _MAX_MARKDOWN_BYTES: + logger.warning( + "Markdown content exceeds %d bytes; truncating", + _MAX_MARKDOWN_BYTES, + ) + content = content[:_MAX_MARKDOWN_BYTES] triples: list[UKOTriple] = [] doc_uri = _doc_uri(resource_uri) diff --git a/src/cleveragents/domain/models/acms/postgresql_analyzer.py b/src/cleveragents/domain/models/acms/postgresql_analyzer.py index 9f9f6b2b7..e66670be2 100644 --- a/src/cleveragents/domain/models/acms/postgresql_analyzer.py +++ b/src/cleveragents/domain/models/acms/postgresql_analyzer.py @@ -32,6 +32,8 @@ from cleveragents.domain.models.acms._postgresql_helpers import ( CREATE_TABLE_RE, CREATE_VIEW_RE, extract_body, + find_unquoted_semicolon, + ident_pair, parse_table_body, schema_uri, strip_sql_comments, @@ -44,6 +46,9 @@ __all__ = ["PostgreSQLAnalyzer"] logger = logging.getLogger(__name__) +# Maximum DDL content size in bytes (S1 fix — DoS guard). +_MAX_DDL_BYTES = 5_242_880 # 5 MiB + class PostgreSQLAnalyzer: """Regex-based PostgreSQL DDL analyzer producing UKO triples. @@ -97,6 +102,15 @@ class PostgreSQLAnalyzer: if not resource_uri or not resource_uri.strip(): raise ValueError("resource_uri must not be empty.") + # Size guard: reject oversized inputs (S1 fix). + if len(content.encode("utf-8")) > _MAX_DDL_BYTES: + logger.warning( + "PostgreSQLAnalyzer: content exceeds %d byte limit; skipping '%s'", + _MAX_DDL_BYTES, + resource_uri, + ) + return [] + content = strip_sql_comments(content) triples: list[UKOTriple] = [] @@ -130,7 +144,7 @@ class PostgreSQLAnalyzer: triples: list[UKOTriple] = [] for match in CREATE_SCHEMA_RE.finditer(content): - s_name = match.group(1).lower() + s_name = (ident_pair(match, 1, 2) or "").lower() if s_name in schemas_seen: continue schemas_seen.add(s_name) @@ -198,8 +212,8 @@ class PostgreSQLAnalyzer: tables_seen = set() for match in CREATE_TABLE_RE.finditer(content): - s_name = match.group(1) or "" - t_name = match.group(2) + s_name = ident_pair(match, 1, 2) or "" + t_name = ident_pair(match, 3, 4) or "" schema_lower = s_name.lower() if s_name else "" t_uri = table_uri(resource_uri, t_name, schema_lower) @@ -257,9 +271,9 @@ class PostgreSQLAnalyzer: triples: list[UKOTriple] = [] for match in CREATE_VIEW_RE.finditer(content): - s_name = match.group(1) or "" + s_name = ident_pair(match, 1, 2) or "" schema_lower = s_name.lower() if s_name else "" - v_name = match.group(2) + v_name = ident_pair(match, 3, 4) or "" v_uri = view_uri(resource_uri, v_name, schema_lower) triples.append( @@ -278,7 +292,7 @@ class PostgreSQLAnalyzer: ) as_start = match.end() - semi_pos = content.find(";", as_start) + semi_pos = find_unquoted_semicolon(content, as_start) view_sql = ( content[as_start:semi_pos].strip() if semi_pos != -1 diff --git a/src/cleveragents/domain/models/core/__init__.py b/src/cleveragents/domain/models/core/__init__.py index b1e2d2127..19016bd81 100644 --- a/src/cleveragents/domain/models/core/__init__.py +++ b/src/cleveragents/domain/models/core/__init__.py @@ -211,6 +211,14 @@ from cleveragents.domain.models.core.project_legacy import ( ProjectSettings, ProjectStats, ) + +# Repo indexing domain models (#195) +from cleveragents.domain.models.core.repo_index import ( + FileRecord, + IndexMetadata, + IndexStatus, + RepoIndex, +) from cleveragents.domain.models.core.resource import ( PhysVirt, Resource, @@ -367,6 +375,7 @@ __all__ = [ "ErrorRecoveryPolicy", "EscalationDecision", "ExecutionEnvironment", + "FileRecord", "FragmentProvenance", "GuardResult", "GuardrailAuditEntry", @@ -376,6 +385,8 @@ __all__ = [ "HistoricalOutcome", "InMemoryChangeSetStore", "InMemoryInvocationTracker", + "IndexMetadata", + "IndexStatus", "InvalidJobTransitionError", "Invariant", "InvariantEnforcementRecord", @@ -425,6 +436,7 @@ __all__ = [ "ProjectStats", "RecoveryAction", "RecoveryHint", + "RepoIndex", "ResolvedToolEntry", "Resource", "ResourceAccessMode", diff --git a/src/cleveragents/domain/models/core/repo_index.py b/src/cleveragents/domain/models/core/repo_index.py new file mode 100644 index 000000000..105e7f17c --- /dev/null +++ b/src/cleveragents/domain/models/core/repo_index.py @@ -0,0 +1,268 @@ +"""Repository indexing domain models for CleverAgents. + +Supports repository indexing for 10K+ file projects with incremental refresh +and language detection. Index metadata and per-file records are persisted +to enable fast incremental re-indexing (only changed files are re-processed). + +## Models + +- :class:`IndexStatus` — indexing lifecycle states +- :class:`FileRecord` — per-file index entry with content hash +- :class:`IndexMetadata` — per-resource index summary +- :class:`RepoIndex` — aggregate: metadata + all file records + +Based on: + +- ``docs/specification.md`` — ``index.*`` config keys (lines 28649-28664) +- ``docs/specification.md`` — ``project link-resource`` indexing output + (lines 2829-2916) +- ``docs/specification.md`` — ``project show`` indexing_status + (lines 3322-3395) +- ``docs/specification.md`` — project data model context configuration + (lines 19719-19727) +- ``docs/specification.md`` — lazy sandboxing vs indexing + (line 24687: "resources are indexed immediately when registered") +- Issue #195: feat(context): add repo indexing service +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from ulid import ULID + +# ULID is 26 characters, Crockford's base32 (matches plan.py pattern) +ULID_PATTERN = r"^[0-9A-HJKMNP-TV-Z]{26}$" + +__all__ = [ + "ULID_PATTERN", + "FileRecord", + "IndexMetadata", + "IndexStatus", + "RepoIndex", +] + +# --------------------------------------------------------------------------- +# Enums +# --------------------------------------------------------------------------- + + +class IndexStatus(StrEnum): + """Lifecycle states for a repository index. + + | State | Description | + |-------------|----------------------------------------------| + | ``pending`` | Index requested but not yet started | + | ``indexing``| Indexing in progress | + | ``ready`` | Index complete and up-to-date | + | ``stale`` | Files changed since last index; refresh needed| + | ``error`` | Indexing failed | + """ + + PENDING = "pending" + INDEXING = "indexing" + READY = "ready" + STALE = "stale" # Planned: set when source files change post-index + ERROR = "error" + + +# --------------------------------------------------------------------------- +# FileRecord — per-file index entry +# --------------------------------------------------------------------------- + + +class FileRecord(BaseModel): + """A single indexed file within a repository resource. + + Stores the content hash to enable incremental refresh: on re-index, + only files whose hash differs from the stored value are re-processed. + + Based on specification.md project data model (lines 19719-19727). + """ + + path: str = Field( + ..., + min_length=1, + max_length=1024, + description="Relative path within the resource root", + ) + # Lowercase hex only — SHA-256 from hashlib is always lowercase. + content_hash: str = Field( + ..., + min_length=64, + max_length=64, + pattern=r"^[0-9a-f]{64}$", + description="SHA-256 hex digest of file content", + ) + token_count: int = Field( + ..., + ge=0, + description="Estimated token count for the file", + ) + language: str = Field( + default="unknown", + max_length=50, + description="Detected programming language (e.g., 'python', 'typescript')", + ) + size_bytes: int = Field( + ..., + ge=0, + description="File size in bytes", + ) + last_modified: datetime = Field( + ..., + description="Last modification timestamp (UTC)", + ) + + @field_validator("path", mode="after") + @classmethod + def _validate_path(cls, v: str) -> str: + """Reject absolute paths and directory-traversal sequences (C5 fix).""" + if v.startswith("/") or v.startswith("\\"): + raise ValueError("path must be relative, not absolute") + if ".." in v.split("/"): + raise ValueError("path must not contain '..' traversal segments") + return v + + @field_validator("last_modified", mode="after") + @classmethod + def _ensure_utc(cls, v: datetime) -> datetime: + """Ensure the timestamp is UTC-aware (convert if needed).""" + if v.tzinfo is None: + return v.replace(tzinfo=UTC) + return v.astimezone(UTC) + + model_config = ConfigDict(frozen=True) + + +# --------------------------------------------------------------------------- +# IndexMetadata — per-resource index summary +# --------------------------------------------------------------------------- + + +class IndexMetadata(BaseModel): + """Summary metadata for an indexed resource. + + Persisted alongside the :class:`FileRecord` entries. Displayed by + ``agents project show`` (spec lines 3322-3395) and used to determine + whether incremental refresh is needed. + + Based on specification.md: + - ``indexing_status`` output (lines 3322-3395): text_index, vector_index, + graph_store, indexed_files, last_indexed_at + - ``project link-resource`` output (lines 2829-2916): status, files_found, + language, estimated_time + """ + + index_id: str = Field( + default_factory=lambda: str(ULID()), + pattern=ULID_PATTERN, + description="Unique index identifier (ULID)", + ) + resource_id: str = Field( + ..., + pattern=ULID_PATTERN, + description="ULID of the indexed resource", + ) + indexed_at: datetime = Field( + default_factory=lambda: datetime.now(tz=UTC), + description="Timestamp when indexing completed (UTC)", + ) + file_count: int = Field( + ..., + ge=0, + description="Total number of indexed files", + ) + token_estimate: int = Field( + ..., + ge=0, + description="Total estimated tokens across all indexed files", + ) + primary_language: str = Field( + default="unknown", + max_length=50, + description=( + "Primary detected language (e.g., 'python'). " + "Based on spec line 2834: 'Language: Python (primary)'" + ), + ) + status: IndexStatus = Field( + default=IndexStatus.PENDING, + description="Current indexing lifecycle status", + ) + error_message: str | None = Field( + default=None, + description="Error details when status is 'error'", + ) + created_at: datetime = Field( + default_factory=lambda: datetime.now(tz=UTC), + description="Timestamp when the index row was first created (UTC)", + ) + + @field_validator("indexed_at", "created_at", mode="after") + @classmethod + def _ensure_utc(cls, v: datetime) -> datetime: + """Ensure the timestamp is UTC-aware (convert if needed).""" + if v.tzinfo is None: + return v.replace(tzinfo=UTC) + return v.astimezone(UTC) + + @model_validator(mode="after") + def _check_error_message_consistency(self) -> IndexMetadata: + """Ensure error_message is only set when status is ERROR (C4 fix).""" + if self.error_message is not None and self.status != IndexStatus.ERROR: + raise ValueError("error_message must be None when status is not 'error'") + return self + + model_config = ConfigDict(frozen=True) + + +# --------------------------------------------------------------------------- +# RepoIndex — aggregate +# --------------------------------------------------------------------------- + + +class RepoIndex(BaseModel): + """Complete index for a single repository resource. + + Aggregates :class:`IndexMetadata` with all :class:`FileRecord` entries. + The ``files`` field uses a tuple for immutability (per implement.md NFR). + + This is the primary return type of + :meth:`RepoIndexingService.index_resource` and + :meth:`RepoIndexingService.refresh_index`. + """ + + metadata: IndexMetadata = Field( + ..., + description="Index summary metadata", + ) + files: tuple[FileRecord, ...] = Field( + default=(), + description="All indexed file records (immutable tuple)", + ) + + @model_validator(mode="after") + def _check_file_count_invariant(self) -> RepoIndex: + """Ensure len(files) matches metadata.file_count (C6 fix).""" + if len(self.files) != self.metadata.file_count: + raise ValueError( + f"len(files)={len(self.files)} != " + f"metadata.file_count={self.metadata.file_count}" + ) + return self + + @model_validator(mode="after") + def _check_token_estimate_invariant(self) -> RepoIndex: + """Ensure token_estimate == sum(token_count) across files.""" + actual = sum(f.token_count for f in self.files) + if actual != self.metadata.token_estimate: + raise ValueError( + f"sum(token_count)={actual} != " + f"metadata.token_estimate={self.metadata.token_estimate}" + ) + return self + + model_config = ConfigDict(frozen=True) diff --git a/src/cleveragents/infrastructure/database/models.py b/src/cleveragents/infrastructure/database/models.py index 80d271680..3c6dc3ead 100644 --- a/src/cleveragents/infrastructure/database/models.py +++ b/src/cleveragents/infrastructure/database/models.py @@ -22,6 +22,8 @@ Alembic migrations. | ``decisions`` | ``DecisionModel`` | Decision tree nodes | | ``decision_dependencies`` | ``DecisionDependencyModel`` | Decision DAG edges | | ``checkpoint_metadata`` | ``CheckpointModel`` | Plan checkpoints | +| ``repo_indexes`` | ``RepoIndexModel`` | Repo index metadata| +| ``indexed_files`` | ``IndexedFileModel`` | Per-file records | Based on ADR-007 (Repository Pattern) and Phase 0 discovery. Includes spec-aligned lifecycle models per Stage A5 @@ -3092,3 +3094,87 @@ class AsyncJobModel(Base): # type: ignore[misc] Index("ix_async_jobs_worker_id", "worker_id"), Index("ix_async_jobs_created_at", "created_at"), ) + + +# --------------------------------------------------------------------------- +# Repo Indexing models (#195) +# --------------------------------------------------------------------------- + + +class RepoIndexModel(Base): + """Per-resource index metadata. + + Stores summary information about an indexed repository resource: + file count, total token estimate, primary language, and status. + One row per indexed resource. + """ + + __tablename__ = "repo_indexes" + + # PK: ULID (26-char string) + index_id = Column(String(26), primary_key=True) + + # FK-like reference to resources (no hard FK — index may outlive resource) + resource_id = Column(String(26), nullable=False, unique=True) + + # Index summary + file_count = Column(Integer, nullable=False, default=0) + token_estimate = Column(Integer, nullable=False, default=0) + primary_language = Column(String(50), nullable=False, default="unknown") + + # Lifecycle status + status = Column(String(20), nullable=False, default="pending") + + # Error details (NULL when not in error state) + error_message = Column(Text, nullable=True) + + # Timestamps (ISO-8601 strings, UTC — 40 chars accommodates microseconds) + indexed_at = Column(String(40), nullable=False) + created_at = Column(String(40), nullable=False) + + __table_args__ = ( + CheckConstraint( + "status IN ('pending', 'indexing', 'ready', 'stale', 'error')", + name="ck_repo_indexes_status", + ), + # N10: resource_id already has unique=True which creates an implicit + # unique index — no need for a redundant explicit index. + Index("ix_repo_indexes_status", "status"), + ) + + +class IndexedFileModel(Base): + """Per-file index record within a repository resource. + + Stores content hash for incremental refresh: on re-index, only files + whose hash differs from the stored value are re-processed. + """ + + __tablename__ = "indexed_files" + + # Composite PK: (index_id, path) + index_id = Column( + String(26), + ForeignKey("repo_indexes.index_id", ondelete="CASCADE"), + primary_key=True, + ) + path = Column(String(1024), primary_key=True) + + # Content fingerprint + content_hash = Column(String(64), nullable=False) + + # Metrics + token_count = Column(Integer, nullable=False, default=0) + size_bytes = Column(Integer, nullable=False, default=0) + + # Language detection + language = Column(String(50), nullable=False, default="unknown") + + # File modification timestamp (ISO-8601 string, UTC — 40 chars for µs) + last_modified = Column(String(40), nullable=False) + + __table_args__ = ( + # index_id is the first column in the composite PK — SQLite uses + # it for lookups already; no separate index needed. + Index("ix_indexed_files_language", "language"), + ) -- 2.52.0