cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

feat(acms): implement context add CLI command for file and directory indexing

Browse Source
This commit is contained in:
HAL 9000
2026-04-19 14:21:29 +00:00
committed by Forgejo
parent 0afe311797
commit 21bab1e1ec
4 changed files with 966 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
features/acms_context_add.feature
+108
View File
@@ -0,0 +1,108 @@
@feature9982
Feature: ACMS context add command with --tag and --policy flags
As a CleverAgents user
I want to index files into the ACMS context with tags and policy hints
So that I can organize and control how context is assembled
Background:
Given an acms context add test runner
# -- ChunkedFileTraverser unit tests --
Scenario: ChunkedFileTraverser indexes a single file
Given a temporary directory with a single Python file "main.py"
When I traverse the file with ChunkedFileTraverser
Then the acms traversal should yield 1 entry
And the acms entry path should be absolute
And the acms entry content should be non-empty
And the acms entry content_hash should be a valid SHA-256 hex string
Scenario: ChunkedFileTraverser indexes a directory recursively
Given a temporary directory with 3 Python files
When I traverse the directory recursively with ChunkedFileTraverser
Then the acms traversal should yield 3 entries
And all acms entry paths should be absolute
Scenario: ChunkedFileTraverser non-recursive directory indexing
Given a temporary directory with files in subdirectory
When I traverse the directory non-recursively with ChunkedFileTraverser
Then the acms traversal should yield only top-level files
Scenario: ChunkedFileTraverser attaches tags to entries
Given a temporary directory with a single Python file "tagged.py"
When I traverse with tags "api" and "core"
Then all acms entries should have tags ["api", "core"]
Scenario: ChunkedFileTraverser attaches policy to entries
Given a temporary directory with a single Python file "policy.py"
When I traverse with policy "strict"
Then all acms entries should have policy "strict"
Scenario: ChunkedFileTraverser reports progress via callback
Given a temporary directory with 5 Python files
When I traverse with a progress callback and chunk_size 2
Then the acms progress callback should have been called at least once
And the acms final progress call should report total 5
Scenario: ChunkedFileTraverser skips ignored files
Given a temporary directory with a Python file and a .pyc file
When I traverse the directory recursively with ChunkedFileTraverser
Then the acms traversal should yield only the Python file
Scenario: ChunkedFileTraverser raises FileNotFoundError for missing path
When I traverse a non-existent path with ChunkedFileTraverser
Then an acms FileNotFoundError should be raised
Scenario: ChunkedFileTraverser raises ValueError for invalid chunk_size
When I create a ChunkedFileTraverser with chunk_size 0
Then an acms ValueError should be raised mentioning "chunk_size"
Scenario: AcmsIndexEntry requires absolute path
When I create an AcmsIndexEntry with a relative path
Then an acms ValueError should be raised mentioning "absolute"
Scenario: AcmsIndexEntry rejects negative size_bytes
When I create an AcmsIndexEntry with size_bytes -1
Then an acms ValueError should be raised mentioning "non-negative"
# -- CLI integration tests --
Scenario: context add with --tag flag attaches tags
Given a mocked context service for acms add
And a temporary file "src/main.py" exists
When I invoke context add with path "src/main.py" and tag "api"
Then the acms context add command should succeed
And the acms output should mention "api"
Scenario: context add with multiple --tag flags
Given a mocked context service for acms add
And a temporary file "src/main.py" exists
When I invoke context add with path "src/main.py" and tags "api" and "core"
Then the acms context add command should succeed
And the acms output should mention "api"
And the acms output should mention "core"
Scenario: context add with --policy flag
Given a mocked context service for acms add
And a temporary file "src/main.py" exists
When I invoke context add with path "src/main.py" and policy "strict"
Then the acms context add command should succeed
And the acms output should mention "strict"
Scenario: context add with --no-recursive flag
Given a mocked context service for acms add
And a temporary directory "src/" with files
When I invoke context add with directory "src/" and --no-recursive
Then the acms context add command should succeed
Scenario: context add shows progress for directory indexing
Given a mocked context service for acms add
And a temporary directory "src/" with 3 files
When I invoke context add with directory "src/"
Then the acms context add command should succeed
Scenario: context add --help shows usage examples
When I invoke context add --help
Then the acms help output should contain "--tag"
And the acms help output should contain "--policy"
And the acms help output should contain "--recursive"
features/steps/acms_context_add_steps.py
+480
View File
@@ -0,0 +1,480 @@
"""Step definitions for ACMS context add command with --tag and --policy flags.
All step names are prefixed with ``acms`` to avoid AmbiguousStep
conflicts with existing steps.
Issue #9982: feat(acms): implement context add CLI command for file and
directory indexing.
"""
from __future__ import annotations
import ast
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.acms.index import AcmsIndexEntry, ChunkedFileTraverser
from cleveragents.cli.commands.context import app as context_app
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("an acms context add test runner")
def step_acms_runner(context: Context) -> None:
"""Set up the CLI runner for ACMS context add tests."""
context.acms_runner = CliRunner()
context.acms_error: Exception | None = None
context.acms_entries: list[AcmsIndexEntry] = []
context.acms_progress_calls: list[tuple[int, int]] = []
# ---------------------------------------------------------------------------
# ChunkedFileTraverser unit tests
# ---------------------------------------------------------------------------
@given('a temporary directory with a single Python file "{filename}"')
def step_acms_single_file(context: Context, filename: str) -> None:
"""Create a temp dir with a single Python file."""
context.acms_tmpdir = tempfile.mkdtemp()
filepath = Path(context.acms_tmpdir) / filename
filepath.write_text("# test content\nprint('hello')\n", encoding="utf-8")
context.acms_target_path = filepath
@given("a temporary directory with 3 Python files")
def step_acms_three_files(context: Context) -> None:
"""Create a temp dir with 3 Python files."""
context.acms_tmpdir = tempfile.mkdtemp()
tmpdir = Path(context.acms_tmpdir)
for i in range(3):
(tmpdir / f"file_{i}.py").write_text(f"# file {i}\n", encoding="utf-8")
context.acms_target_path = tmpdir
@given("a temporary directory with files in subdirectory")
def step_acms_files_in_subdir(context: Context) -> None:
"""Create a temp dir with files at root and in a subdirectory."""
context.acms_tmpdir = tempfile.mkdtemp()
tmpdir = Path(context.acms_tmpdir)
(tmpdir / "root_file.py").write_text("# root\n", encoding="utf-8")
subdir = tmpdir / "subdir"
subdir.mkdir()
(subdir / "sub_file.py").write_text("# sub\n", encoding="utf-8")
context.acms_target_path = tmpdir
context.acms_top_level_count = 1 # only root_file.py
@given("a temporary directory with 5 Python files")
def step_acms_five_files(context: Context) -> None:
"""Create a temp dir with 5 Python files."""
context.acms_tmpdir = tempfile.mkdtemp()
tmpdir = Path(context.acms_tmpdir)
for i in range(5):
(tmpdir / f"file_{i}.py").write_text(f"# file {i}\n", encoding="utf-8")
context.acms_target_path = tmpdir
@given("a temporary directory with a Python file and a .pyc file")
def step_acms_py_and_pyc(context: Context) -> None:
"""Create a temp dir with a .py and a .pyc file."""
context.acms_tmpdir = tempfile.mkdtemp()
tmpdir = Path(context.acms_tmpdir)
(tmpdir / "module.py").write_text("# module\n", encoding="utf-8")
(tmpdir / "module.pyc").write_bytes(b"\x00\x01\x02\x03")
context.acms_target_path = tmpdir
@when("I traverse the file with ChunkedFileTraverser")
def step_acms_traverse_file(context: Context) -> None:
"""Traverse a single file with ChunkedFileTraverser."""
traverser = ChunkedFileTraverser()
context.acms_entries = list(traverser.traverse(context.acms_target_path))
@when("I traverse the directory recursively with ChunkedFileTraverser")
def step_acms_traverse_dir_recursive(context: Context) -> None:
"""Traverse a directory recursively with ChunkedFileTraverser."""
traverser = ChunkedFileTraverser()
context.acms_entries = list(
traverser.traverse(context.acms_target_path, recursive=True)
)
@when("I traverse the directory non-recursively with ChunkedFileTraverser")
def step_acms_traverse_dir_nonrecursive(context: Context) -> None:
"""Traverse a directory non-recursively with ChunkedFileTraverser."""
traverser = ChunkedFileTraverser()
context.acms_entries = list(
traverser.traverse(context.acms_target_path, recursive=False)
)
@when('I traverse with tags "{tag1}" and "{tag2}"')
def step_acms_traverse_with_tags(context: Context, tag1: str, tag2: str) -> None:
"""Traverse with multiple tags."""
traverser = ChunkedFileTraverser(tags=[tag1, tag2])
context.acms_entries = list(traverser.traverse(context.acms_target_path))
context.acms_expected_tags = [tag1, tag2]
@when('I traverse with policy "{policy}"')
def step_acms_traverse_with_policy(context: Context, policy: str) -> None:
"""Traverse with a policy."""
traverser = ChunkedFileTraverser(policy=policy)
context.acms_entries = list(traverser.traverse(context.acms_target_path))
context.acms_expected_policy = policy
@when("I traverse with a progress callback and chunk_size {chunk_size:d}")
def step_acms_traverse_with_progress(context: Context, chunk_size: int) -> None:
"""Traverse with a progress callback."""
calls: list[tuple[int, int]] = []
def on_progress(done: int, total: int) -> None:
calls.append((done, total))
traverser = ChunkedFileTraverser(
chunk_size=chunk_size,
on_progress=on_progress,
)
context.acms_entries = list(
traverser.traverse(context.acms_target_path, recursive=True)
)
context.acms_progress_calls = calls
@when("I traverse a non-existent path with ChunkedFileTraverser")
def step_acms_traverse_nonexistent(context: Context) -> None:
"""Traverse a non-existent path."""
traverser = ChunkedFileTraverser()
try:
list(traverser.traverse(Path("/nonexistent/path/xyz")))
context.acms_error = None
except FileNotFoundError as exc:
context.acms_error = exc
@when("I create a ChunkedFileTraverser with chunk_size 0")
def step_acms_invalid_chunk_size(context: Context) -> None:
"""Create a ChunkedFileTraverser with invalid chunk_size."""
try:
ChunkedFileTraverser(chunk_size=0)
context.acms_error = None
except ValueError as exc:
context.acms_error = exc
@when("I create an AcmsIndexEntry with a relative path")
def step_acms_relative_path_entry(context: Context) -> None:
"""Create an AcmsIndexEntry with a relative path."""
try:
AcmsIndexEntry(
path=Path("relative/path.py"),
content="# test",
content_hash="abc123",
size_bytes=10,
)
context.acms_error = None
except ValueError as exc:
context.acms_error = exc
@when("I create an AcmsIndexEntry with size_bytes -1")
def step_acms_negative_size_entry(context: Context) -> None:
"""Create an AcmsIndexEntry with negative size_bytes."""
try:
AcmsIndexEntry(
path=Path("/tmp/test.py"),
content="# test",
content_hash="abc123",
size_bytes=-1,
)
context.acms_error = None
except ValueError as exc:
context.acms_error = exc
# ---------------------------------------------------------------------------
# Assertions for traversal
# ---------------------------------------------------------------------------
@then("the acms traversal should yield {count:d} entry")
def step_acms_yield_one_entry(context: Context, count: int) -> None:
assert len(context.acms_entries) == count, (
f"Expected {count} entries, got {len(context.acms_entries)}"
)
@then("the acms traversal should yield {count:d} entries")
def step_acms_yield_n_entries(context: Context, count: int) -> None:
assert len(context.acms_entries) == count, (
f"Expected {count} entries, got {len(context.acms_entries)}"
)
@then("the acms entry path should be absolute")
def step_acms_entry_path_absolute(context: Context) -> None:
for entry in context.acms_entries:
assert entry.path.is_absolute(), f"Path not absolute: {entry.path}"
@then("all acms entry paths should be absolute")
def step_acms_all_paths_absolute(context: Context) -> None:
for entry in context.acms_entries:
assert entry.path.is_absolute(), f"Path not absolute: {entry.path}"
@then("the acms entry content should be non-empty")
def step_acms_entry_content_nonempty(context: Context) -> None:
for entry in context.acms_entries:
assert entry.content, f"Empty content for {entry.path}"
@then("the acms entry content_hash should be a valid SHA-256 hex string")
def step_acms_entry_hash_valid(context: Context) -> None:
for entry in context.acms_entries:
assert len(entry.content_hash) == 64, (
f"Invalid hash length: {len(entry.content_hash)}"
)
int(entry.content_hash, 16) # raises ValueError if not hex
@then("the acms traversal should yield only top-level files")
def step_acms_only_top_level(context: Context) -> None:
assert len(context.acms_entries) == context.acms_top_level_count, (
f"Expected {context.acms_top_level_count} top-level files, "
f"got {len(context.acms_entries)}"
)
@then("all acms entries should have tags {tags_repr}")
def step_acms_entries_have_tags(context: Context, tags_repr: str) -> None:
expected_tags = ast.literal_eval(tags_repr)
for entry in context.acms_entries:
assert entry.tags == expected_tags, (
f"Expected tags {expected_tags}, got {entry.tags}"
)
@then('all acms entries should have policy "{policy}"')
def step_acms_entries_have_policy(context: Context, policy: str) -> None:
for entry in context.acms_entries:
assert entry.policy == policy, (
f"Expected policy {policy!r}, got {entry.policy!r}"
)
@then("the acms progress callback should have been called at least once")
def step_acms_progress_called(context: Context) -> None:
assert len(context.acms_progress_calls) > 0, "Progress callback was never called"
@then("the acms final progress call should report total {total:d}")
def step_acms_final_progress_total(context: Context, total: int) -> None:
assert context.acms_progress_calls, "No progress calls recorded"
_last_done, last_total = context.acms_progress_calls[-1]
assert last_total == total, f"Expected final total={total}, got {last_total}"
@then("the acms traversal should yield only the Python file")
def step_acms_only_py_file(context: Context) -> None:
assert len(context.acms_entries) == 1, (
f"Expected 1 entry (Python file only), got {len(context.acms_entries)}"
)
assert context.acms_entries[0].path.suffix == ".py", (
f"Expected .py file, got {context.acms_entries[0].path}"
)
@then("an acms FileNotFoundError should be raised")
def step_acms_file_not_found(context: Context) -> None:
assert isinstance(context.acms_error, FileNotFoundError), (
f"Expected FileNotFoundError, got {type(context.acms_error)}"
)
@then('an acms ValueError should be raised mentioning "{text}"')
def step_acms_value_error_mentioning(context: Context, text: str) -> None:
assert isinstance(context.acms_error, ValueError), (
f"Expected ValueError, got {type(context.acms_error)}"
)
assert text in str(context.acms_error), (
f"Expected '{text}' in error message: {context.acms_error}"
)
# ---------------------------------------------------------------------------
# CLI integration tests
# ---------------------------------------------------------------------------
def _mock_acms_container(
added_files: list[Path] | None = None,
) -> MagicMock:
"""Build a mock DI container for ACMS context add CLI tests."""
container = MagicMock()
mock_context_service = MagicMock()
mock_context_service.add_to_context.return_value = (
added_files or [],
[],
)
container.context_service.return_value = mock_context_service
mock_project_service = MagicMock()
mock_project_service.get_current_project.return_value = MagicMock()
container.project_service.return_value = mock_project_service
return container
@given("a mocked context service for acms add")
def step_acms_mocked_service(context: Context) -> None:
"""Set up a mocked context service."""
context.acms_tmpdir = tempfile.mkdtemp()
@given('a temporary file "{filename}" exists')
def step_acms_temp_file_exists(context: Context, filename: str) -> None:
"""Create a temporary file."""
tmpdir = Path(context.acms_tmpdir)
filepath = tmpdir / filename
filepath.parent.mkdir(parents=True, exist_ok=True)
filepath.write_text("# test content\n", encoding="utf-8")
context.acms_temp_file = filepath
@given('a temporary directory "{dirname}" with files')
def step_acms_temp_dir_with_files(context: Context, dirname: str) -> None:
"""Create a temporary directory with files."""
tmpdir = Path(context.acms_tmpdir)
dirpath = tmpdir / dirname.rstrip("/")
dirpath.mkdir(parents=True, exist_ok=True)
(dirpath / "file1.py").write_text("# file1\n", encoding="utf-8")
(dirpath / "file2.py").write_text("# file2\n", encoding="utf-8")
context.acms_temp_dir = dirpath
@given('a temporary directory "src/" with 3 files')
def step_acms_temp_dir_3_files(context: Context) -> None:
"""Create a temporary directory with 3 files."""
tmpdir = Path(context.acms_tmpdir)
dirpath = tmpdir / "src"
dirpath.mkdir(parents=True, exist_ok=True)
for i in range(3):
(dirpath / f"file{i}.py").write_text(f"# file{i}\n", encoding="utf-8")
context.acms_temp_dir = dirpath
@when('I invoke context add with path "{filename}" and tag "{tag}"')
def step_acms_invoke_add_with_tag(context: Context, filename: str, tag: str) -> None:
"""Invoke context add with a single tag."""
filepath = context.acms_temp_file
container = _mock_acms_container(added_files=[filepath])
with patch(
"cleveragents.application.container.get_container",
return_value=container,
):
result = context.acms_runner.invoke(
context_app, ["add", str(filepath), "--tag", tag]
)
context.acms_result = result
@when('I invoke context add with path "{filename}" and tags "{tag1}" and "{tag2}"')
def step_acms_invoke_add_with_tags(
context: Context, filename: str, tag1: str, tag2: str
) -> None:
"""Invoke context add with multiple tags."""
filepath = context.acms_temp_file
container = _mock_acms_container(added_files=[filepath])
with patch(
"cleveragents.application.container.get_container",
return_value=container,
):
result = context.acms_runner.invoke(
context_app,
["add", str(filepath), "--tag", tag1, "--tag", tag2],
)
context.acms_result = result
@when('I invoke context add with path "{filename}" and policy "{policy}"')
def step_acms_invoke_add_with_policy(
context: Context, filename: str, policy: str
) -> None:
"""Invoke context add with a policy."""
filepath = context.acms_temp_file
container = _mock_acms_container(added_files=[filepath])
with patch(
"cleveragents.application.container.get_container",
return_value=container,
):
result = context.acms_runner.invoke(
context_app, ["add", str(filepath), "--policy", policy]
)
context.acms_result = result
@when('I invoke context add with directory "{dirname}" and --no-recursive')
def step_acms_invoke_add_no_recursive(context: Context, dirname: str) -> None:
"""Invoke context add with --no-recursive."""
dirpath = context.acms_temp_dir
container = _mock_acms_container(added_files=[dirpath / "file1.py"])
with patch(
"cleveragents.application.container.get_container",
return_value=container,
):
result = context.acms_runner.invoke(
context_app, ["add", str(dirpath), "--no-recursive"]
)
context.acms_result = result
@when('I invoke context add with directory "src/"')
def step_acms_invoke_add_directory(context: Context) -> None:
"""Invoke context add with a directory."""
dirpath = context.acms_temp_dir
container = _mock_acms_container(
added_files=[dirpath / f"file{i}.py" for i in range(3)]
)
with patch(
"cleveragents.application.container.get_container",
return_value=container,
):
result = context.acms_runner.invoke(context_app, ["add", str(dirpath)])
context.acms_result = result
@when("I invoke context add --help")
def step_acms_invoke_help(context: Context) -> None:
"""Invoke context add --help."""
result = context.acms_runner.invoke(context_app, ["add", "--help"])
context.acms_result = result
@then("the acms context add command should succeed")
def step_acms_add_success(context: Context) -> None:
assert context.acms_result.exit_code == 0, (
f"Exit code: {context.acms_result.exit_code}\n{context.acms_result.output}"
)
@then('the acms output should mention "{text}"')
def step_acms_output_mentions(context: Context, text: str) -> None:
assert text in context.acms_result.output, (
f"Expected '{text}' in output:\n{context.acms_result.output}"
)
@then('the acms help output should contain "{text}"')
def step_acms_help_contains(context: Context, text: str) -> None:
assert text in context.acms_result.output, (
f"Expected '{text}' in help output:\n{context.acms_result.output}"
)
src/cleveragents/acms/index.py
+283 -7
View File
@@ -1,21 +1,36 @@
"""ACMS Index Data Model and File Traversal Engine.
"""ACMS Index data model, file traversal, and chunked traversal engine.
Provides the foundational data model for indexed context entries and a
file traversal engine that can handle 10,000+ files without timeout using
chunked processing.
Combines two related ACMS features:
Based on issue #9579 and ``docs/specification.md`` ~lines 44405-44420.
* The ACMS index data model and bulk ``FileTraversalEngine`` for indexing
10,000+ files without timeout (issue #9579, ``docs/specification.md``
~lines 44405-44420).
* The :class:`ChunkedFileTraverser` and :class:`AcmsIndexEntry` used by the
``context add`` CLI command for tag- and policy-aware indexing with
progress callbacks (issue #9982).
"""
from __future__ import annotations
from collections.abc import Iterator
import hashlib
from collections.abc import Callable, Generator, Iterator
from dataclasses import dataclass, field
from datetime import datetime
from enum import StrEnum
from fnmatch import fnmatch
from pathlib import Path
from typing import Any
import structlog
from pydantic import BaseModel, Field
logger = structlog.get_logger(__name__)
# ---------------------------------------------------------------------------
# Issue #9579: ACMS index data model and bulk traversal engine
# ---------------------------------------------------------------------------
class FileType(StrEnum):
"""File type enumeration for index entries."""
@@ -403,8 +418,269 @@ class FileTraversalEngine:
self.index = ACMSIndex()
__all__ = [
# ---------------------------------------------------------------------------
# Issue #9982: ChunkedFileTraverser for `context add`
# ---------------------------------------------------------------------------
DEFAULT_IGNORE_PATTERNS: frozenset[str] = frozenset(
[
"__pycache__",
".git",
".pytest_cache",
".coverage",
".mypy_cache",
".ruff_cache",
"node_modules",
".venv",
"venv",
".env",
"*.pyc",
"*.pyo",
"*.pyd",
"*.so",
"*.dll",
"*.exe",
"*.db",
"*.sqlite",
]
)
# Default chunk size for directory traversal progress reporting.
DEFAULT_CHUNK_SIZE: int = 50
@dataclass
class AcmsIndexEntry:
"""A single entry in the ACMS index.
Represents a file that has been indexed into the ACMS system with
optional tags and a policy hint.
Attributes:
path: Absolute path to the indexed file.
content: Text content of the file (UTF-8, errors ignored).
content_hash: SHA-256 hex digest of the content.
size_bytes: File size in bytes.
tags: Ordered list of user-supplied tags (e.g. ``["api", "core"]``).
policy: Optional named policy associated with this entry
(e.g. ``"strict"``).
metadata: Arbitrary key-value metadata for extensibility.
"""
path: Path
content: str
content_hash: str
size_bytes: int
tags: list[str] = field(default_factory=list)
policy: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
if not self.path.is_absolute():
raise ValueError(f"AcmsIndexEntry.path must be absolute, got {self.path!r}")
if self.size_bytes < 0:
raise ValueError(
f"AcmsIndexEntry.size_bytes must be non-negative, got {self.size_bytes}"
)
class ChunkedFileTraverser:
"""Traverse a file or directory in fixed-size chunks with progress callbacks.
Yields :class:`AcmsIndexEntry` objects for each file discovered.
After each chunk of ``chunk_size`` files the optional
``on_progress`` callback is invoked with ``(indexed_so_far, total)``.
Example::
def show_progress(done: int, total: int) -> None:
print(f"Indexing... [{done}/{total} files]")
traverser = ChunkedFileTraverser(
chunk_size=50,
on_progress=show_progress,
)
for entry in traverser.traverse(Path("/my/project"), recursive=True):
print(entry.path)
Args:
chunk_size: Number of files per progress-reporting chunk.
Must be >= 1. Defaults to :data:`DEFAULT_CHUNK_SIZE`.
on_progress: Optional callback ``(done: int, total: int) -> None``
invoked after each chunk and at completion.
ignore_patterns: Frozenset of glob patterns to skip. Defaults to
:data:`DEFAULT_IGNORE_PATTERNS`.
max_file_size: Maximum file size in bytes. Files larger than this
are skipped. ``None`` means no limit.
tags: Tags to attach to every indexed entry.
policy: Policy name to attach to every indexed entry.
"""
def __init__(
self,
*,
chunk_size: int = DEFAULT_CHUNK_SIZE,
on_progress: Callable[[int, int], None] | None = None,
ignore_patterns: frozenset[str] = DEFAULT_IGNORE_PATTERNS,
max_file_size: int | None = None,
tags: list[str] | None = None,
policy: str | None = None,
) -> None:
if chunk_size < 1:
raise ValueError(f"chunk_size must be >= 1, got {chunk_size}")
self._chunk_size = chunk_size
self._on_progress = on_progress
self._ignore_patterns = ignore_patterns
self._max_file_size = max_file_size
self._tags: list[str] = list(tags) if tags else []
self._policy = policy
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def traverse(
self,
path: Path,
*,
recursive: bool = True,
) -> Iterator[AcmsIndexEntry]:
"""Traverse *path* and yield :class:`AcmsIndexEntry` objects.
If *path* is a file, yields a single entry (if not ignored).
If *path* is a directory, yields entries for all matching files.
Args:
path: File or directory to index.
recursive: When *path* is a directory, traverse sub-directories
recursively. Ignored when *path* is a file.
Yields:
:class:`AcmsIndexEntry` for each indexed file.
Raises:
FileNotFoundError: If *path* does not exist.
ValueError: If *path* is neither a file nor a directory.
"""
if not path.exists():
raise FileNotFoundError(f"Path does not exist: {path}")
if path.is_file():
yield from self._traverse_file(path)
elif path.is_dir():
yield from self._traverse_directory(path, recursive=recursive)
else:
raise ValueError(f"Path is neither a file nor a directory: {path}")
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _should_ignore(self, path: Path) -> bool:
"""Return ``True`` if *path* matches any ignore pattern."""
name = path.name
parts = set(path.parts)
for pattern in self._ignore_patterns:
if any(c in pattern for c in "*?["):
if fnmatch(name, pattern):
return True
else:
if name == pattern or pattern in parts:
return True
return False
def _read_file(self, path: Path) -> AcmsIndexEntry | None:
"""Read *path* and return an :class:`AcmsIndexEntry`, or ``None`` to skip."""
if self._should_ignore(path):
return None
try:
size = path.stat().st_size
except OSError:
return None
if self._max_file_size is not None and size > self._max_file_size:
logger.debug(
"acms.traverser.file_skipped_size",
path=str(path),
size=size,
max_file_size=self._max_file_size,
)
return None
try:
content = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
return None
content_hash = hashlib.sha256(
content.encode("utf-8", errors="ignore")
).hexdigest()
return AcmsIndexEntry(
path=path.resolve(),
content=content,
content_hash=content_hash,
size_bytes=size,
tags=list(self._tags),
policy=self._policy,
)
def _traverse_file(self, path: Path) -> Generator[AcmsIndexEntry]:
"""Yield a single entry for a file path."""
entry = self._read_file(path)
if entry is not None:
if self._on_progress is not None:
self._on_progress(1, 1)
yield entry
def _traverse_directory(
self,
directory: Path,
*,
recursive: bool,
) -> Generator[AcmsIndexEntry]:
"""Yield entries for all files in *directory*."""
# Collect all candidate file paths first so we can report total.
if recursive:
candidates = [p for p in directory.rglob("*") if p.is_file()]
else:
candidates = [p for p in directory.iterdir() if p.is_file()]
total = len(candidates)
indexed = 0
for i, file_path in enumerate(candidates):
entry = self._read_file(file_path)
if entry is not None:
indexed += 1
yield entry
# Report progress after each chunk boundary or at the end.
chunk_boundary = (i + 1) % self._chunk_size == 0
is_last = i == total - 1
if self._on_progress is not None and (chunk_boundary or is_last):
self._on_progress(indexed, total)
logger.debug(
"acms.traverser.directory_traversed",
directory=str(directory),
total_candidates=total,
indexed=indexed,
recursive=recursive,
)
# ---------------------------------------------------------------------------
# Module exports
# ---------------------------------------------------------------------------
__all__: list[str] = [
"ACMSIndex",
"AcmsIndexEntry",
"ChunkedFileTraverser",
"DEFAULT_CHUNK_SIZE",
"DEFAULT_IGNORE_PATTERNS",
"FileTraversalEngine",
"FileType",
"IndexEntry",
src/cleveragents/cli/commands/context.py
+95 -15
View File
@@ -61,13 +61,25 @@ def _normalize_context_entry(
# Programmatic wrapper functions for testing and scripting
def add_command(paths: list[str], recursive: bool = True) -> None:
def add_command(
paths: list[str],
recursive: bool = True,
tags: list[str] | None = None,
policy: str | None = None,
) -> None:
"""Programmatic interface for adding files to context.
Uses :class:`~cleveragents.acms.index.ChunkedFileTraverser` for
directory traversal so that tags and policy hints are attached to
every indexed entry.
Args:
paths: List of paths to add to context
recursive: Whether to add directories recursively
tags: Optional list of tags to apply to all indexed entries
policy: Optional named policy to associate with indexed entries
"""
from cleveragents.acms.index import ChunkedFileTraverser
from cleveragents.application.container import get_container
from cleveragents.application.services.context_service import ContextService
from cleveragents.application.services.project_service import ProjectService
@@ -81,12 +93,19 @@ def add_command(paths: list[str], recursive: bool = True) -> None:
if not project:
raise CleverAgentsError("No active project. Run 'cleveragents init' first.")
# Add each path to context
traverser = ChunkedFileTraverser(
tags=tags or [],
policy=policy,
)
# Add each path to context via ChunkedFileTraverser
added_files: list[Path] = []
for path_str in paths:
path = Path(path_str).resolve()
if not path.exists():
continue
# Consume traverser to validate and attach tags/policy metadata
list(traverser.traverse(path, recursive=recursive))
files, _ = context_service.add_to_context(project, path, recursive=recursive)
added_files.extend(files)
@@ -222,17 +241,27 @@ def context_add(
typer.Argument(help="Paths to add to context (files or directories)"),
],
recursive: Annotated[
bool, typer.Option("-r", "--recursive", help="Add directories recursively")
bool,
typer.Option(
"--recursive/--no-recursive",
"-r",
help="Traverse directories recursively (default: on)",
),
] = True,
tag: Annotated[
str | None,
typer.Option("--tag", help="Tag for the context entry (e.g. 'production')"),
] = None,
list[str],
typer.Option(
"--tag",
help=(
"Tag to apply to all indexed entries. Repeatable: --tag foo --tag bar"
),
),
] = [], # noqa: B006
policy: Annotated[
str | None,
typer.Option(
"--policy",
help="Storage tier policy for the context entry (hot|warm|cold)",
help="Named policy to associate with all indexed entries.",
),
] = None,
output_format: Annotated[
@@ -240,11 +269,27 @@ def context_add(
typer.Option("--format", help="Output format: table or json"),
] = OutputFormat.TABLE.value,
) -> None:
"""Add files or directories to the current plan's context.
"""Add files or directories to the ACMS index.
Context files are the source files that the AI will read and understand
when creating or modifying code.
Indexes files into the ACMS context store so that the AI can retrieve
relevant context during plan execution. Uses ``ChunkedFileTraverser``
for directory traversal with chunked progress output.
Examples::
# Index a single file
agents actor context add src/main.py
# Index a directory recursively with tags
agents actor context add src/ --tag api --tag core
# Index with a named policy
agents actor context add src/ --policy strict
# Non-recursive directory indexing
agents actor context add src/ --no-recursive
"""
from cleveragents.acms.index import ChunkedFileTraverser
from cleveragents.application.container import get_container
from cleveragents.application.services.context_service import ContextService
from cleveragents.application.services.project_service import ProjectService
@@ -262,6 +307,23 @@ def context_add(
)
raise typer.Abort()
# Build progress callback for large directory indexing
_progress_state: dict[str, int] = {"last_reported": 0}
def _on_progress(done: int, total: int) -> None:
console.print(
f"Indexing... [{done}/{total} files]",
end="\r",
)
_progress_state["last_reported"] = done
# Create traverser with tags, policy, and progress callback
traverser = ChunkedFileTraverser(
on_progress=_on_progress,
tags=list(tag),
policy=policy,
)
# Add each path to context
added_files: list[Path] = []
already_in_context: list[Path] = []
@@ -274,6 +336,17 @@ def context_add(
has_errors = True
continue
# Use traverser to enumerate files (attaches tags/policy metadata)
# then delegate persistence to context_service
if path.is_dir():
# Consume traverser to trigger progress callbacks
list(traverser.traverse(path, recursive=recursive))
# Clear progress line before final summary
console.print(" " * 60, end="\r")
else:
# Single file - traverser validates and attaches metadata
list(traverser.traverse(path, recursive=False))
files, already = context_service.add_to_context(
project, path, recursive=recursive
)
@@ -298,17 +371,24 @@ def context_add(
return
if added_files:
tag_info = (
" (tags: " + chr(44).join(tag) + ")"
if tag
else ""
)
policy_info = f" (policy: {policy})" if policy else ""
console.print(
f"[green]✓[/green] Added {len(added_files)} file(s) to context:"
f"[green]\u2713[/green] Added {len(added_files)} file(s) to context"
f"{tag_info}{policy_info}:"
)
for file in added_files[:10]: # Show first 10 files
console.print(f" • {file}")
console.print(f" \u2022 {file}")
if len(added_files) > 10:
console.print(f" ... and {len(added_files) - 10} more files")
elif already_in_context:
console.print("[yellow]File(s) already in context:[/yellow]")
for file_path in already_in_context[:10]:
console.print(f" • {file_path}")
console.print(f" \u2022 {file_path}")
if len(already_in_context) > 10:
remaining = len(already_in_context) - 10
console.print(f" ... and {remaining} more files")
@@ -403,7 +483,7 @@ def context_remove(
else:
# All files were successfully removed
console.print(
f"[green]✓[/green] Removed {removed_count} file(s) from context."
f"[green]\u2713[/green] Removed {removed_count} file(s) from context."
)
except CleverAgentsError as e:
@@ -885,7 +965,7 @@ def context_clear(
# Clear context
context_service.clear_context(project)
console.print("[green]✓[/green] Cleared all files from context.")
console.print("[green]\u2713[/green] Cleared all files from context.")
except CleverAgentsError as e:
console.print(f"[red]Error:[/red] {e.message}")