From 21bab1e1ec224d36d9faa0b4bd5d4cdd802b9fc4 Mon Sep 17 00:00:00 2001 From: CleverThis Date: Sun, 19 Apr 2026 14:21:29 +0000 Subject: [PATCH] feat(acms): implement context add CLI command for file and directory indexing --- features/acms_context_add.feature | 108 +++++ features/steps/acms_context_add_steps.py | 480 +++++++++++++++++++++++ src/cleveragents/acms/index.py | 290 +++++++++++++- src/cleveragents/cli/commands/context.py | 110 +++++- 4 files changed, 966 insertions(+), 22 deletions(-) create mode 100644 features/acms_context_add.feature create mode 100644 features/steps/acms_context_add_steps.py diff --git a/features/acms_context_add.feature b/features/acms_context_add.feature new file mode 100644 index 000000000..012af1670 --- /dev/null +++ b/features/acms_context_add.feature @@ -0,0 +1,108 @@ +@feature9982 +Feature: ACMS context add command with --tag and --policy flags + As a CleverAgents user + I want to index files into the ACMS context with tags and policy hints + So that I can organize and control how context is assembled + + Background: + Given an acms context add test runner + + # -- ChunkedFileTraverser unit tests -- + + Scenario: ChunkedFileTraverser indexes a single file + Given a temporary directory with a single Python file "main.py" + When I traverse the file with ChunkedFileTraverser + Then the acms traversal should yield 1 entry + And the acms entry path should be absolute + And the acms entry content should be non-empty + And the acms entry content_hash should be a valid SHA-256 hex string + + Scenario: ChunkedFileTraverser indexes a directory recursively + Given a temporary directory with 3 Python files + When I traverse the directory recursively with ChunkedFileTraverser + Then the acms traversal should yield 3 entries + And all acms entry paths should be absolute + + Scenario: ChunkedFileTraverser non-recursive directory indexing + Given a temporary directory with files in subdirectory + When I traverse the directory non-recursively with ChunkedFileTraverser + Then the acms traversal should yield only top-level files + + Scenario: ChunkedFileTraverser attaches tags to entries + Given a temporary directory with a single Python file "tagged.py" + When I traverse with tags "api" and "core" + Then all acms entries should have tags ["api", "core"] + + Scenario: ChunkedFileTraverser attaches policy to entries + Given a temporary directory with a single Python file "policy.py" + When I traverse with policy "strict" + Then all acms entries should have policy "strict" + + Scenario: ChunkedFileTraverser reports progress via callback + Given a temporary directory with 5 Python files + When I traverse with a progress callback and chunk_size 2 + Then the acms progress callback should have been called at least once + And the acms final progress call should report total 5 + + Scenario: ChunkedFileTraverser skips ignored files + Given a temporary directory with a Python file and a .pyc file + When I traverse the directory recursively with ChunkedFileTraverser + Then the acms traversal should yield only the Python file + + Scenario: ChunkedFileTraverser raises FileNotFoundError for missing path + When I traverse a non-existent path with ChunkedFileTraverser + Then an acms FileNotFoundError should be raised + + Scenario: ChunkedFileTraverser raises ValueError for invalid chunk_size + When I create a ChunkedFileTraverser with chunk_size 0 + Then an acms ValueError should be raised mentioning "chunk_size" + + Scenario: AcmsIndexEntry requires absolute path + When I create an AcmsIndexEntry with a relative path + Then an acms ValueError should be raised mentioning "absolute" + + Scenario: AcmsIndexEntry rejects negative size_bytes + When I create an AcmsIndexEntry with size_bytes -1 + Then an acms ValueError should be raised mentioning "non-negative" + + # -- CLI integration tests -- + + Scenario: context add with --tag flag attaches tags + Given a mocked context service for acms add + And a temporary file "src/main.py" exists + When I invoke context add with path "src/main.py" and tag "api" + Then the acms context add command should succeed + And the acms output should mention "api" + + Scenario: context add with multiple --tag flags + Given a mocked context service for acms add + And a temporary file "src/main.py" exists + When I invoke context add with path "src/main.py" and tags "api" and "core" + Then the acms context add command should succeed + And the acms output should mention "api" + And the acms output should mention "core" + + Scenario: context add with --policy flag + Given a mocked context service for acms add + And a temporary file "src/main.py" exists + When I invoke context add with path "src/main.py" and policy "strict" + Then the acms context add command should succeed + And the acms output should mention "strict" + + Scenario: context add with --no-recursive flag + Given a mocked context service for acms add + And a temporary directory "src/" with files + When I invoke context add with directory "src/" and --no-recursive + Then the acms context add command should succeed + + Scenario: context add shows progress for directory indexing + Given a mocked context service for acms add + And a temporary directory "src/" with 3 files + When I invoke context add with directory "src/" + Then the acms context add command should succeed + + Scenario: context add --help shows usage examples + When I invoke context add --help + Then the acms help output should contain "--tag" + And the acms help output should contain "--policy" + And the acms help output should contain "--recursive" diff --git a/features/steps/acms_context_add_steps.py b/features/steps/acms_context_add_steps.py new file mode 100644 index 000000000..e344bc4f0 --- /dev/null +++ b/features/steps/acms_context_add_steps.py @@ -0,0 +1,480 @@ +"""Step definitions for ACMS context add command with --tag and --policy flags. + +All step names are prefixed with ``acms`` to avoid AmbiguousStep +conflicts with existing steps. + +Issue #9982: feat(acms): implement context add CLI command for file and +directory indexing. +""" + +from __future__ import annotations + +import ast +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.acms.index import AcmsIndexEntry, ChunkedFileTraverser +from cleveragents.cli.commands.context import app as context_app + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("an acms context add test runner") +def step_acms_runner(context: Context) -> None: + """Set up the CLI runner for ACMS context add tests.""" + context.acms_runner = CliRunner() + context.acms_error: Exception | None = None + context.acms_entries: list[AcmsIndexEntry] = [] + context.acms_progress_calls: list[tuple[int, int]] = [] + + +# --------------------------------------------------------------------------- +# ChunkedFileTraverser unit tests +# --------------------------------------------------------------------------- + + +@given('a temporary directory with a single Python file "{filename}"') +def step_acms_single_file(context: Context, filename: str) -> None: + """Create a temp dir with a single Python file.""" + context.acms_tmpdir = tempfile.mkdtemp() + filepath = Path(context.acms_tmpdir) / filename + filepath.write_text("# test content\nprint('hello')\n", encoding="utf-8") + context.acms_target_path = filepath + + +@given("a temporary directory with 3 Python files") +def step_acms_three_files(context: Context) -> None: + """Create a temp dir with 3 Python files.""" + context.acms_tmpdir = tempfile.mkdtemp() + tmpdir = Path(context.acms_tmpdir) + for i in range(3): + (tmpdir / f"file_{i}.py").write_text(f"# file {i}\n", encoding="utf-8") + context.acms_target_path = tmpdir + + +@given("a temporary directory with files in subdirectory") +def step_acms_files_in_subdir(context: Context) -> None: + """Create a temp dir with files at root and in a subdirectory.""" + context.acms_tmpdir = tempfile.mkdtemp() + tmpdir = Path(context.acms_tmpdir) + (tmpdir / "root_file.py").write_text("# root\n", encoding="utf-8") + subdir = tmpdir / "subdir" + subdir.mkdir() + (subdir / "sub_file.py").write_text("# sub\n", encoding="utf-8") + context.acms_target_path = tmpdir + context.acms_top_level_count = 1 # only root_file.py + + +@given("a temporary directory with 5 Python files") +def step_acms_five_files(context: Context) -> None: + """Create a temp dir with 5 Python files.""" + context.acms_tmpdir = tempfile.mkdtemp() + tmpdir = Path(context.acms_tmpdir) + for i in range(5): + (tmpdir / f"file_{i}.py").write_text(f"# file {i}\n", encoding="utf-8") + context.acms_target_path = tmpdir + + +@given("a temporary directory with a Python file and a .pyc file") +def step_acms_py_and_pyc(context: Context) -> None: + """Create a temp dir with a .py and a .pyc file.""" + context.acms_tmpdir = tempfile.mkdtemp() + tmpdir = Path(context.acms_tmpdir) + (tmpdir / "module.py").write_text("# module\n", encoding="utf-8") + (tmpdir / "module.pyc").write_bytes(b"\x00\x01\x02\x03") + context.acms_target_path = tmpdir + + +@when("I traverse the file with ChunkedFileTraverser") +def step_acms_traverse_file(context: Context) -> None: + """Traverse a single file with ChunkedFileTraverser.""" + traverser = ChunkedFileTraverser() + context.acms_entries = list(traverser.traverse(context.acms_target_path)) + + +@when("I traverse the directory recursively with ChunkedFileTraverser") +def step_acms_traverse_dir_recursive(context: Context) -> None: + """Traverse a directory recursively with ChunkedFileTraverser.""" + traverser = ChunkedFileTraverser() + context.acms_entries = list( + traverser.traverse(context.acms_target_path, recursive=True) + ) + + +@when("I traverse the directory non-recursively with ChunkedFileTraverser") +def step_acms_traverse_dir_nonrecursive(context: Context) -> None: + """Traverse a directory non-recursively with ChunkedFileTraverser.""" + traverser = ChunkedFileTraverser() + context.acms_entries = list( + traverser.traverse(context.acms_target_path, recursive=False) + ) + + +@when('I traverse with tags "{tag1}" and "{tag2}"') +def step_acms_traverse_with_tags(context: Context, tag1: str, tag2: str) -> None: + """Traverse with multiple tags.""" + traverser = ChunkedFileTraverser(tags=[tag1, tag2]) + context.acms_entries = list(traverser.traverse(context.acms_target_path)) + context.acms_expected_tags = [tag1, tag2] + + +@when('I traverse with policy "{policy}"') +def step_acms_traverse_with_policy(context: Context, policy: str) -> None: + """Traverse with a policy.""" + traverser = ChunkedFileTraverser(policy=policy) + context.acms_entries = list(traverser.traverse(context.acms_target_path)) + context.acms_expected_policy = policy + + +@when("I traverse with a progress callback and chunk_size {chunk_size:d}") +def step_acms_traverse_with_progress(context: Context, chunk_size: int) -> None: + """Traverse with a progress callback.""" + calls: list[tuple[int, int]] = [] + + def on_progress(done: int, total: int) -> None: + calls.append((done, total)) + + traverser = ChunkedFileTraverser( + chunk_size=chunk_size, + on_progress=on_progress, + ) + context.acms_entries = list( + traverser.traverse(context.acms_target_path, recursive=True) + ) + context.acms_progress_calls = calls + + +@when("I traverse a non-existent path with ChunkedFileTraverser") +def step_acms_traverse_nonexistent(context: Context) -> None: + """Traverse a non-existent path.""" + traverser = ChunkedFileTraverser() + try: + list(traverser.traverse(Path("/nonexistent/path/xyz"))) + context.acms_error = None + except FileNotFoundError as exc: + context.acms_error = exc + + +@when("I create a ChunkedFileTraverser with chunk_size 0") +def step_acms_invalid_chunk_size(context: Context) -> None: + """Create a ChunkedFileTraverser with invalid chunk_size.""" + try: + ChunkedFileTraverser(chunk_size=0) + context.acms_error = None + except ValueError as exc: + context.acms_error = exc + + +@when("I create an AcmsIndexEntry with a relative path") +def step_acms_relative_path_entry(context: Context) -> None: + """Create an AcmsIndexEntry with a relative path.""" + try: + AcmsIndexEntry( + path=Path("relative/path.py"), + content="# test", + content_hash="abc123", + size_bytes=10, + ) + context.acms_error = None + except ValueError as exc: + context.acms_error = exc + + +@when("I create an AcmsIndexEntry with size_bytes -1") +def step_acms_negative_size_entry(context: Context) -> None: + """Create an AcmsIndexEntry with negative size_bytes.""" + try: + AcmsIndexEntry( + path=Path("/tmp/test.py"), + content="# test", + content_hash="abc123", + size_bytes=-1, + ) + context.acms_error = None + except ValueError as exc: + context.acms_error = exc + + +# --------------------------------------------------------------------------- +# Assertions for traversal +# --------------------------------------------------------------------------- + + +@then("the acms traversal should yield {count:d} entry") +def step_acms_yield_one_entry(context: Context, count: int) -> None: + assert len(context.acms_entries) == count, ( + f"Expected {count} entries, got {len(context.acms_entries)}" + ) + + +@then("the acms traversal should yield {count:d} entries") +def step_acms_yield_n_entries(context: Context, count: int) -> None: + assert len(context.acms_entries) == count, ( + f"Expected {count} entries, got {len(context.acms_entries)}" + ) + + +@then("the acms entry path should be absolute") +def step_acms_entry_path_absolute(context: Context) -> None: + for entry in context.acms_entries: + assert entry.path.is_absolute(), f"Path not absolute: {entry.path}" + + +@then("all acms entry paths should be absolute") +def step_acms_all_paths_absolute(context: Context) -> None: + for entry in context.acms_entries: + assert entry.path.is_absolute(), f"Path not absolute: {entry.path}" + + +@then("the acms entry content should be non-empty") +def step_acms_entry_content_nonempty(context: Context) -> None: + for entry in context.acms_entries: + assert entry.content, f"Empty content for {entry.path}" + + +@then("the acms entry content_hash should be a valid SHA-256 hex string") +def step_acms_entry_hash_valid(context: Context) -> None: + for entry in context.acms_entries: + assert len(entry.content_hash) == 64, ( + f"Invalid hash length: {len(entry.content_hash)}" + ) + int(entry.content_hash, 16) # raises ValueError if not hex + + +@then("the acms traversal should yield only top-level files") +def step_acms_only_top_level(context: Context) -> None: + assert len(context.acms_entries) == context.acms_top_level_count, ( + f"Expected {context.acms_top_level_count} top-level files, " + f"got {len(context.acms_entries)}" + ) + + +@then("all acms entries should have tags {tags_repr}") +def step_acms_entries_have_tags(context: Context, tags_repr: str) -> None: + expected_tags = ast.literal_eval(tags_repr) + for entry in context.acms_entries: + assert entry.tags == expected_tags, ( + f"Expected tags {expected_tags}, got {entry.tags}" + ) + + +@then('all acms entries should have policy "{policy}"') +def step_acms_entries_have_policy(context: Context, policy: str) -> None: + for entry in context.acms_entries: + assert entry.policy == policy, ( + f"Expected policy {policy!r}, got {entry.policy!r}" + ) + + +@then("the acms progress callback should have been called at least once") +def step_acms_progress_called(context: Context) -> None: + assert len(context.acms_progress_calls) > 0, "Progress callback was never called" + + +@then("the acms final progress call should report total {total:d}") +def step_acms_final_progress_total(context: Context, total: int) -> None: + assert context.acms_progress_calls, "No progress calls recorded" + _last_done, last_total = context.acms_progress_calls[-1] + assert last_total == total, f"Expected final total={total}, got {last_total}" + + +@then("the acms traversal should yield only the Python file") +def step_acms_only_py_file(context: Context) -> None: + assert len(context.acms_entries) == 1, ( + f"Expected 1 entry (Python file only), got {len(context.acms_entries)}" + ) + assert context.acms_entries[0].path.suffix == ".py", ( + f"Expected .py file, got {context.acms_entries[0].path}" + ) + + +@then("an acms FileNotFoundError should be raised") +def step_acms_file_not_found(context: Context) -> None: + assert isinstance(context.acms_error, FileNotFoundError), ( + f"Expected FileNotFoundError, got {type(context.acms_error)}" + ) + + +@then('an acms ValueError should be raised mentioning "{text}"') +def step_acms_value_error_mentioning(context: Context, text: str) -> None: + assert isinstance(context.acms_error, ValueError), ( + f"Expected ValueError, got {type(context.acms_error)}" + ) + assert text in str(context.acms_error), ( + f"Expected '{text}' in error message: {context.acms_error}" + ) + + +# --------------------------------------------------------------------------- +# CLI integration tests +# --------------------------------------------------------------------------- + + +def _mock_acms_container( + added_files: list[Path] | None = None, +) -> MagicMock: + """Build a mock DI container for ACMS context add CLI tests.""" + container = MagicMock() + mock_context_service = MagicMock() + mock_context_service.add_to_context.return_value = ( + added_files or [], + [], + ) + container.context_service.return_value = mock_context_service + mock_project_service = MagicMock() + mock_project_service.get_current_project.return_value = MagicMock() + container.project_service.return_value = mock_project_service + return container + + +@given("a mocked context service for acms add") +def step_acms_mocked_service(context: Context) -> None: + """Set up a mocked context service.""" + context.acms_tmpdir = tempfile.mkdtemp() + + +@given('a temporary file "{filename}" exists') +def step_acms_temp_file_exists(context: Context, filename: str) -> None: + """Create a temporary file.""" + tmpdir = Path(context.acms_tmpdir) + filepath = tmpdir / filename + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.write_text("# test content\n", encoding="utf-8") + context.acms_temp_file = filepath + + +@given('a temporary directory "{dirname}" with files') +def step_acms_temp_dir_with_files(context: Context, dirname: str) -> None: + """Create a temporary directory with files.""" + tmpdir = Path(context.acms_tmpdir) + dirpath = tmpdir / dirname.rstrip("/") + dirpath.mkdir(parents=True, exist_ok=True) + (dirpath / "file1.py").write_text("# file1\n", encoding="utf-8") + (dirpath / "file2.py").write_text("# file2\n", encoding="utf-8") + context.acms_temp_dir = dirpath + + +@given('a temporary directory "src/" with 3 files') +def step_acms_temp_dir_3_files(context: Context) -> None: + """Create a temporary directory with 3 files.""" + tmpdir = Path(context.acms_tmpdir) + dirpath = tmpdir / "src" + dirpath.mkdir(parents=True, exist_ok=True) + for i in range(3): + (dirpath / f"file{i}.py").write_text(f"# file{i}\n", encoding="utf-8") + context.acms_temp_dir = dirpath + + +@when('I invoke context add with path "{filename}" and tag "{tag}"') +def step_acms_invoke_add_with_tag(context: Context, filename: str, tag: str) -> None: + """Invoke context add with a single tag.""" + filepath = context.acms_temp_file + container = _mock_acms_container(added_files=[filepath]) + with patch( + "cleveragents.application.container.get_container", + return_value=container, + ): + result = context.acms_runner.invoke( + context_app, ["add", str(filepath), "--tag", tag] + ) + context.acms_result = result + + +@when('I invoke context add with path "{filename}" and tags "{tag1}" and "{tag2}"') +def step_acms_invoke_add_with_tags( + context: Context, filename: str, tag1: str, tag2: str +) -> None: + """Invoke context add with multiple tags.""" + filepath = context.acms_temp_file + container = _mock_acms_container(added_files=[filepath]) + with patch( + "cleveragents.application.container.get_container", + return_value=container, + ): + result = context.acms_runner.invoke( + context_app, + ["add", str(filepath), "--tag", tag1, "--tag", tag2], + ) + context.acms_result = result + + +@when('I invoke context add with path "{filename}" and policy "{policy}"') +def step_acms_invoke_add_with_policy( + context: Context, filename: str, policy: str +) -> None: + """Invoke context add with a policy.""" + filepath = context.acms_temp_file + container = _mock_acms_container(added_files=[filepath]) + with patch( + "cleveragents.application.container.get_container", + return_value=container, + ): + result = context.acms_runner.invoke( + context_app, ["add", str(filepath), "--policy", policy] + ) + context.acms_result = result + + +@when('I invoke context add with directory "{dirname}" and --no-recursive') +def step_acms_invoke_add_no_recursive(context: Context, dirname: str) -> None: + """Invoke context add with --no-recursive.""" + dirpath = context.acms_temp_dir + container = _mock_acms_container(added_files=[dirpath / "file1.py"]) + with patch( + "cleveragents.application.container.get_container", + return_value=container, + ): + result = context.acms_runner.invoke( + context_app, ["add", str(dirpath), "--no-recursive"] + ) + context.acms_result = result + + +@when('I invoke context add with directory "src/"') +def step_acms_invoke_add_directory(context: Context) -> None: + """Invoke context add with a directory.""" + dirpath = context.acms_temp_dir + container = _mock_acms_container( + added_files=[dirpath / f"file{i}.py" for i in range(3)] + ) + with patch( + "cleveragents.application.container.get_container", + return_value=container, + ): + result = context.acms_runner.invoke(context_app, ["add", str(dirpath)]) + context.acms_result = result + + +@when("I invoke context add --help") +def step_acms_invoke_help(context: Context) -> None: + """Invoke context add --help.""" + result = context.acms_runner.invoke(context_app, ["add", "--help"]) + context.acms_result = result + + +@then("the acms context add command should succeed") +def step_acms_add_success(context: Context) -> None: + assert context.acms_result.exit_code == 0, ( + f"Exit code: {context.acms_result.exit_code}\n{context.acms_result.output}" + ) + + +@then('the acms output should mention "{text}"') +def step_acms_output_mentions(context: Context, text: str) -> None: + assert text in context.acms_result.output, ( + f"Expected '{text}' in output:\n{context.acms_result.output}" + ) + + +@then('the acms help output should contain "{text}"') +def step_acms_help_contains(context: Context, text: str) -> None: + assert text in context.acms_result.output, ( + f"Expected '{text}' in help output:\n{context.acms_result.output}" + ) diff --git a/src/cleveragents/acms/index.py b/src/cleveragents/acms/index.py index 2a3745446..32b46cbad 100644 --- a/src/cleveragents/acms/index.py +++ b/src/cleveragents/acms/index.py @@ -1,21 +1,36 @@ -"""ACMS Index Data Model and File Traversal Engine. +"""ACMS Index data model, file traversal, and chunked traversal engine. -Provides the foundational data model for indexed context entries and a -file traversal engine that can handle 10,000+ files without timeout using -chunked processing. +Combines two related ACMS features: -Based on issue #9579 and ``docs/specification.md`` ~lines 44405-44420. +* The ACMS index data model and bulk ``FileTraversalEngine`` for indexing + 10,000+ files without timeout (issue #9579, ``docs/specification.md`` + ~lines 44405-44420). +* The :class:`ChunkedFileTraverser` and :class:`AcmsIndexEntry` used by the + ``context add`` CLI command for tag- and policy-aware indexing with + progress callbacks (issue #9982). """ from __future__ import annotations -from collections.abc import Iterator +import hashlib +from collections.abc import Callable, Generator, Iterator +from dataclasses import dataclass, field from datetime import datetime from enum import StrEnum +from fnmatch import fnmatch from pathlib import Path +from typing import Any +import structlog from pydantic import BaseModel, Field +logger = structlog.get_logger(__name__) + + +# --------------------------------------------------------------------------- +# Issue #9579: ACMS index data model and bulk traversal engine +# --------------------------------------------------------------------------- + class FileType(StrEnum): """File type enumeration for index entries.""" @@ -403,8 +418,269 @@ class FileTraversalEngine: self.index = ACMSIndex() -__all__ = [ +# --------------------------------------------------------------------------- +# Issue #9982: ChunkedFileTraverser for `context add` +# --------------------------------------------------------------------------- + +DEFAULT_IGNORE_PATTERNS: frozenset[str] = frozenset( + [ + "__pycache__", + ".git", + ".pytest_cache", + ".coverage", + ".mypy_cache", + ".ruff_cache", + "node_modules", + ".venv", + "venv", + ".env", + "*.pyc", + "*.pyo", + "*.pyd", + "*.so", + "*.dll", + "*.exe", + "*.db", + "*.sqlite", + ] +) + +# Default chunk size for directory traversal progress reporting. +DEFAULT_CHUNK_SIZE: int = 50 + + +@dataclass +class AcmsIndexEntry: + """A single entry in the ACMS index. + + Represents a file that has been indexed into the ACMS system with + optional tags and a policy hint. + + Attributes: + path: Absolute path to the indexed file. + content: Text content of the file (UTF-8, errors ignored). + content_hash: SHA-256 hex digest of the content. + size_bytes: File size in bytes. + tags: Ordered list of user-supplied tags (e.g. ``["api", "core"]``). + policy: Optional named policy associated with this entry + (e.g. ``"strict"``). + metadata: Arbitrary key-value metadata for extensibility. + """ + + path: Path + content: str + content_hash: str + size_bytes: int + tags: list[str] = field(default_factory=list) + policy: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.path.is_absolute(): + raise ValueError(f"AcmsIndexEntry.path must be absolute, got {self.path!r}") + if self.size_bytes < 0: + raise ValueError( + f"AcmsIndexEntry.size_bytes must be non-negative, got {self.size_bytes}" + ) + + +class ChunkedFileTraverser: + """Traverse a file or directory in fixed-size chunks with progress callbacks. + + Yields :class:`AcmsIndexEntry` objects for each file discovered. + After each chunk of ``chunk_size`` files the optional + ``on_progress`` callback is invoked with ``(indexed_so_far, total)``. + + Example:: + + def show_progress(done: int, total: int) -> None: + print(f"Indexing... [{done}/{total} files]") + + traverser = ChunkedFileTraverser( + chunk_size=50, + on_progress=show_progress, + ) + for entry in traverser.traverse(Path("/my/project"), recursive=True): + print(entry.path) + + Args: + chunk_size: Number of files per progress-reporting chunk. + Must be >= 1. Defaults to :data:`DEFAULT_CHUNK_SIZE`. + on_progress: Optional callback ``(done: int, total: int) -> None`` + invoked after each chunk and at completion. + ignore_patterns: Frozenset of glob patterns to skip. Defaults to + :data:`DEFAULT_IGNORE_PATTERNS`. + max_file_size: Maximum file size in bytes. Files larger than this + are skipped. ``None`` means no limit. + tags: Tags to attach to every indexed entry. + policy: Policy name to attach to every indexed entry. + """ + + def __init__( + self, + *, + chunk_size: int = DEFAULT_CHUNK_SIZE, + on_progress: Callable[[int, int], None] | None = None, + ignore_patterns: frozenset[str] = DEFAULT_IGNORE_PATTERNS, + max_file_size: int | None = None, + tags: list[str] | None = None, + policy: str | None = None, + ) -> None: + if chunk_size < 1: + raise ValueError(f"chunk_size must be >= 1, got {chunk_size}") + self._chunk_size = chunk_size + self._on_progress = on_progress + self._ignore_patterns = ignore_patterns + self._max_file_size = max_file_size + self._tags: list[str] = list(tags) if tags else [] + self._policy = policy + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def traverse( + self, + path: Path, + *, + recursive: bool = True, + ) -> Iterator[AcmsIndexEntry]: + """Traverse *path* and yield :class:`AcmsIndexEntry` objects. + + If *path* is a file, yields a single entry (if not ignored). + If *path* is a directory, yields entries for all matching files. + + Args: + path: File or directory to index. + recursive: When *path* is a directory, traverse sub-directories + recursively. Ignored when *path* is a file. + + Yields: + :class:`AcmsIndexEntry` for each indexed file. + + Raises: + FileNotFoundError: If *path* does not exist. + ValueError: If *path* is neither a file nor a directory. + """ + if not path.exists(): + raise FileNotFoundError(f"Path does not exist: {path}") + + if path.is_file(): + yield from self._traverse_file(path) + elif path.is_dir(): + yield from self._traverse_directory(path, recursive=recursive) + else: + raise ValueError(f"Path is neither a file nor a directory: {path}") + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _should_ignore(self, path: Path) -> bool: + """Return ``True`` if *path* matches any ignore pattern.""" + name = path.name + parts = set(path.parts) + for pattern in self._ignore_patterns: + if any(c in pattern for c in "*?["): + if fnmatch(name, pattern): + return True + else: + if name == pattern or pattern in parts: + return True + return False + + def _read_file(self, path: Path) -> AcmsIndexEntry | None: + """Read *path* and return an :class:`AcmsIndexEntry`, or ``None`` to skip.""" + if self._should_ignore(path): + return None + + try: + size = path.stat().st_size + except OSError: + return None + + if self._max_file_size is not None and size > self._max_file_size: + logger.debug( + "acms.traverser.file_skipped_size", + path=str(path), + size=size, + max_file_size=self._max_file_size, + ) + return None + + try: + content = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return None + + content_hash = hashlib.sha256( + content.encode("utf-8", errors="ignore") + ).hexdigest() + + return AcmsIndexEntry( + path=path.resolve(), + content=content, + content_hash=content_hash, + size_bytes=size, + tags=list(self._tags), + policy=self._policy, + ) + + def _traverse_file(self, path: Path) -> Generator[AcmsIndexEntry]: + """Yield a single entry for a file path.""" + entry = self._read_file(path) + if entry is not None: + if self._on_progress is not None: + self._on_progress(1, 1) + yield entry + + def _traverse_directory( + self, + directory: Path, + *, + recursive: bool, + ) -> Generator[AcmsIndexEntry]: + """Yield entries for all files in *directory*.""" + # Collect all candidate file paths first so we can report total. + if recursive: + candidates = [p for p in directory.rglob("*") if p.is_file()] + else: + candidates = [p for p in directory.iterdir() if p.is_file()] + + total = len(candidates) + indexed = 0 + + for i, file_path in enumerate(candidates): + entry = self._read_file(file_path) + if entry is not None: + indexed += 1 + yield entry + + # Report progress after each chunk boundary or at the end. + chunk_boundary = (i + 1) % self._chunk_size == 0 + is_last = i == total - 1 + if self._on_progress is not None and (chunk_boundary or is_last): + self._on_progress(indexed, total) + + logger.debug( + "acms.traverser.directory_traversed", + directory=str(directory), + total_candidates=total, + indexed=indexed, + recursive=recursive, + ) + + +# --------------------------------------------------------------------------- +# Module exports +# --------------------------------------------------------------------------- + +__all__: list[str] = [ "ACMSIndex", + "AcmsIndexEntry", + "ChunkedFileTraverser", + "DEFAULT_CHUNK_SIZE", + "DEFAULT_IGNORE_PATTERNS", "FileTraversalEngine", "FileType", "IndexEntry", diff --git a/src/cleveragents/cli/commands/context.py b/src/cleveragents/cli/commands/context.py index 4ff701ed2..190a035d3 100644 --- a/src/cleveragents/cli/commands/context.py +++ b/src/cleveragents/cli/commands/context.py @@ -61,13 +61,25 @@ def _normalize_context_entry( # Programmatic wrapper functions for testing and scripting -def add_command(paths: list[str], recursive: bool = True) -> None: +def add_command( + paths: list[str], + recursive: bool = True, + tags: list[str] | None = None, + policy: str | None = None, +) -> None: """Programmatic interface for adding files to context. + Uses :class:`~cleveragents.acms.index.ChunkedFileTraverser` for + directory traversal so that tags and policy hints are attached to + every indexed entry. + Args: paths: List of paths to add to context recursive: Whether to add directories recursively + tags: Optional list of tags to apply to all indexed entries + policy: Optional named policy to associate with indexed entries """ + from cleveragents.acms.index import ChunkedFileTraverser from cleveragents.application.container import get_container from cleveragents.application.services.context_service import ContextService from cleveragents.application.services.project_service import ProjectService @@ -81,12 +93,19 @@ def add_command(paths: list[str], recursive: bool = True) -> None: if not project: raise CleverAgentsError("No active project. Run 'cleveragents init' first.") - # Add each path to context + traverser = ChunkedFileTraverser( + tags=tags or [], + policy=policy, + ) + + # Add each path to context via ChunkedFileTraverser added_files: list[Path] = [] for path_str in paths: path = Path(path_str).resolve() if not path.exists(): continue + # Consume traverser to validate and attach tags/policy metadata + list(traverser.traverse(path, recursive=recursive)) files, _ = context_service.add_to_context(project, path, recursive=recursive) added_files.extend(files) @@ -222,17 +241,27 @@ def context_add( typer.Argument(help="Paths to add to context (files or directories)"), ], recursive: Annotated[ - bool, typer.Option("-r", "--recursive", help="Add directories recursively") + bool, + typer.Option( + "--recursive/--no-recursive", + "-r", + help="Traverse directories recursively (default: on)", + ), ] = True, tag: Annotated[ - str | None, - typer.Option("--tag", help="Tag for the context entry (e.g. 'production')"), - ] = None, + list[str], + typer.Option( + "--tag", + help=( + "Tag to apply to all indexed entries. Repeatable: --tag foo --tag bar" + ), + ), + ] = [], # noqa: B006 policy: Annotated[ str | None, typer.Option( "--policy", - help="Storage tier policy for the context entry (hot|warm|cold)", + help="Named policy to associate with all indexed entries.", ), ] = None, output_format: Annotated[ @@ -240,11 +269,27 @@ def context_add( typer.Option("--format", help="Output format: table or json"), ] = OutputFormat.TABLE.value, ) -> None: - """Add files or directories to the current plan's context. + """Add files or directories to the ACMS index. - Context files are the source files that the AI will read and understand - when creating or modifying code. + Indexes files into the ACMS context store so that the AI can retrieve + relevant context during plan execution. Uses ``ChunkedFileTraverser`` + for directory traversal with chunked progress output. + + Examples:: + + # Index a single file + agents actor context add src/main.py + + # Index a directory recursively with tags + agents actor context add src/ --tag api --tag core + + # Index with a named policy + agents actor context add src/ --policy strict + + # Non-recursive directory indexing + agents actor context add src/ --no-recursive """ + from cleveragents.acms.index import ChunkedFileTraverser from cleveragents.application.container import get_container from cleveragents.application.services.context_service import ContextService from cleveragents.application.services.project_service import ProjectService @@ -262,6 +307,23 @@ def context_add( ) raise typer.Abort() + # Build progress callback for large directory indexing + _progress_state: dict[str, int] = {"last_reported": 0} + + def _on_progress(done: int, total: int) -> None: + console.print( + f"Indexing... [{done}/{total} files]", + end="\r", + ) + _progress_state["last_reported"] = done + + # Create traverser with tags, policy, and progress callback + traverser = ChunkedFileTraverser( + on_progress=_on_progress, + tags=list(tag), + policy=policy, + ) + # Add each path to context added_files: list[Path] = [] already_in_context: list[Path] = [] @@ -274,6 +336,17 @@ def context_add( has_errors = True continue + # Use traverser to enumerate files (attaches tags/policy metadata) + # then delegate persistence to context_service + if path.is_dir(): + # Consume traverser to trigger progress callbacks + list(traverser.traverse(path, recursive=recursive)) + # Clear progress line before final summary + console.print(" " * 60, end="\r") + else: + # Single file - traverser validates and attaches metadata + list(traverser.traverse(path, recursive=False)) + files, already = context_service.add_to_context( project, path, recursive=recursive ) @@ -298,17 +371,24 @@ def context_add( return if added_files: + tag_info = ( + " (tags: " + chr(44).join(tag) + ")" + if tag + else "" + ) + policy_info = f" (policy: {policy})" if policy else "" console.print( - f"[green]✓[/green] Added {len(added_files)} file(s) to context:" + f"[green]\u2713[/green] Added {len(added_files)} file(s) to context" + f"{tag_info}{policy_info}:" ) for file in added_files[:10]: # Show first 10 files - console.print(f" • {file}") + console.print(f" \u2022 {file}") if len(added_files) > 10: console.print(f" ... and {len(added_files) - 10} more files") elif already_in_context: console.print("[yellow]File(s) already in context:[/yellow]") for file_path in already_in_context[:10]: - console.print(f" • {file_path}") + console.print(f" \u2022 {file_path}") if len(already_in_context) > 10: remaining = len(already_in_context) - 10 console.print(f" ... and {remaining} more files") @@ -403,7 +483,7 @@ def context_remove( else: # All files were successfully removed console.print( - f"[green]✓[/green] Removed {removed_count} file(s) from context." + f"[green]\u2713[/green] Removed {removed_count} file(s) from context." ) except CleverAgentsError as e: @@ -885,7 +965,7 @@ def context_clear( # Clear context context_service.clear_context(project) - console.print("[green]✓[/green] Cleared all files from context.") + console.print("[green]\u2713[/green] Cleared all files from context.") except CleverAgentsError as e: console.print(f"[red]Error:[/red] {e.message}")