diff --git a/.gitignore b/.gitignore index 4f40d20e2..e066e2b25 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,6 @@ output.xml report.html .agent-orchestration agents-test + +# Generated test reports (CI artifacts) +test_reports/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b010a42..6807f1be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). failure output significantly easier to spot in CI and local runs. A worker crash (unhandled exception) is detected via an all-zero summary and the captured traceback is always surfaced. +- **Directory Clustering Absolute Path Fix** (#9401): Fixed `DecompositionService._directory_key` to correctly handle absolute file paths by computing relative paths before extracting directory keys. Previously, the function used a fixed depth of 2 path components, causing all absolute paths to collapse into a single bucket (e.g., `/home` for every file on the system), making directory-based clustering completely ineffective. The fix adds an optional `root` parameter to `_directory_key()` and `ClusteringStrategy.cluster_by_directory()`, and updates `DecompositionService._build_hierarchy()` to compute the common root and pass it through, ensuring directory clustering groups paths by their actual directory hierarchy in production use. - **Automation Profile Silent Fallback** (#8232): `_resolve_profile_for_plan` in `PlanLifecycleService` now raises a clear `ValidationError` when a plan's diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 67cfaa955..074e45e6d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -7,7 +7,6 @@ * Jeffrey Phillips Freeman * Luis Mendes * Rui Hu -* HAL 9000 # Details @@ -22,4 +21,6 @@ Below are some of the specific details of various contributions. * Rui Hu has contributed the v3 actor YAML schema validation fix (#5869): added `ActorConfigSchema` validation to the `agents actor add --config` CLI command, covering cycle detection, required field validation, and enum validation for v3 YAML actor definitions. * HAL 9000 has contributed automated bug fixes, including fix #7488 (store sandbox_path in checkpoint metadata to enable rollback). * This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc. +* HAL 9000 has contributed the directory clustering absolute path fix (#9401): updated `_directory_key()` to accept an optional `root` parameter for relative path computation, enabling effective directory-based clustering for projects with absolute file paths. * HAL 9000 has contributed automated bug fixes, CLI output formatting improvements, and ongoing maintenance as part of the CleverAgents automation system. +* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc. diff --git a/features/large_project_decomposition.feature b/features/large_project_decomposition.feature index 5f9e837e8..e006ce210 100644 --- a/features/large_project_decomposition.feature +++ b/features/large_project_decomposition.feature @@ -232,3 +232,19 @@ Feature: Large-project hierarchical decomposition Scenario: Directory key handles short paths When I compute directory key for a single-component path Then the directory key should be empty string + + # --- absolute path clustering ------------------------------------------ + + Scenario: Directory clustering works correctly with absolute file paths + Given a project with absolute paths in distinct subdirectories + When I decompose with directory clustering + Then at least two clusters should have different directory prefixes + + Scenario: _directory_key returns correct key for absolute path with root + When I compute directory key for an absolute path with a root + Then the directory key should reflect the relative path structure + + Scenario: Directory clustering does not collapse all absolute paths into one bucket + Given a project with absolute paths spanning multiple top-level directories + When I decompose with max_files_per_subplan 50 + Then the decomposition result should have max_depth_reached >= 1 diff --git a/features/steps/large_project_decomposition_coverage_steps.py b/features/steps/large_project_decomposition_coverage_steps.py index 744ca804f..cc589b8b9 100644 --- a/features/steps/large_project_decomposition_coverage_steps.py +++ b/features/steps/large_project_decomposition_coverage_steps.py @@ -240,3 +240,66 @@ def step_when_dir_key_short(context: Any) -> None: @then("the directory key should be empty string") def step_then_dir_key_empty(context: Any) -> None: assert context.dir_key == "" + + +# --------------------------------------------------------------------------- +# absolute path clustering scenarios +# --------------------------------------------------------------------------- + + +@given("a project with absolute paths in distinct subdirectories") +def step_given_abs_path_project(context): + import os + import tempfile + + tmpdir = tempfile.mkdtemp(prefix="decompose-abs-") + context.tmpdir = tmpdir + paths = [] + for sub in ("src/api", "src/web"): + dirpath = os.path.join(tmpdir, sub) + os.makedirs(dirpath, exist_ok=True) + for i in range(50): + fpath = os.path.join(dirpath, f"f_{i:03d}.py") + with open(fpath, "w") as fh: + fh.write("x" * 200) + paths.append(fpath) + context.files = sorted(paths) + + +@given("a project with absolute paths spanning multiple top-level directories") +def step_given_abs_path_multi_top(context): + import os + import tempfile + + tmpdir = tempfile.mkdtemp(prefix="decompose-multi-") + context.tmpdir = tmpdir + paths = [] + for top in ("alpha", "beta", "gamma", "delta"): + for sub in ("core", "utils"): + dirpath = os.path.join(tmpdir, top, sub) + os.makedirs(dirpath, exist_ok=True) + for i in range(30): + fpath = os.path.join(dirpath, f"f_{i:03d}.py") + with open(fpath, "w") as fh: + fh.write("x" * 200) + paths.append(fpath) + context.files = sorted(paths) + + +@when("I compute directory key for an absolute path with a root") +def step_when_dir_key_abs_with_root(context): + from cleveragents.application.services.decomposition_clustering import ( + _directory_key, + ) + + root = "/home/user/project" + path = "/home/user/project/src/api/handler.py" + context.dir_key = _directory_key(path, depth=2, root=root) + context.expected_dir_key = "src/api" + + +@then("the directory key should reflect the relative path structure") +def step_then_dir_key_relative(context): + assert context.dir_key == context.expected_dir_key, ( + f"expected {context.expected_dir_key!r}, got {context.dir_key!r}" + ) diff --git a/src/cleveragents/application/services/decomposition_clustering.py b/src/cleveragents/application/services/decomposition_clustering.py index d78a207e3..db2554704 100644 --- a/src/cleveragents/application/services/decomposition_clustering.py +++ b/src/cleveragents/application/services/decomposition_clustering.py @@ -54,14 +54,52 @@ def _extension_of(path: str) -> str: return ext.lower() -def _directory_key(path: str, depth: int = 2) -> str: +def _directory_key(path: str, depth: int = 2, root: str | None = None) -> str: """Return the first *depth* path components as a grouping key. For ``"src/cleveragents/foo/bar.py"`` with *depth=2* the key is ``"src/cleveragents"``. + + When *root* is provided, the key is computed relative to the root. + For example, with root ``"/home/user/project"`` and path + ``"/home/user/project/src/api/handler.py"``, the relative path is + ``"src/api/handler.py"`` and the key is ``"src/api"``. + + Args: + path: File path (absolute or relative). + depth: Number of leading path components for the grouping key. + root: Optional root directory. When provided, the key is computed + relative to this root. + + Returns: + The directory key (first *depth* components of the path). """ - parts = path.replace("\\", "/").split("/") - return "/".join(parts[:depth]) if len(parts) > depth else "/".join(parts[:-1]) + # Normalize path separators + normalized = path.replace("\\", "/") + + # If root is provided, compute relative path + if root: + root_normalized = root.replace("\\", "/") + # Ensure root ends with / for proper prefix matching + if not root_normalized.endswith("/"): + root_normalized += "/" + # Remove root prefix if path starts with it + if normalized.startswith(root_normalized): + normalized = normalized[len(root_normalized) :] + + parts = normalized.split("/") + # Filter out empty parts (from leading / in absolute paths) + parts = [p for p in parts if p] + + if not parts: + return "" + + # Return first *depth* components, or all but the last (filename) + if len(parts) > depth: + return "/".join(parts[:depth]) + else: + # For short paths, return all but the last component (filename) + return "/".join(parts[:-1]) if len(parts) > 1 else "" # --------------------------------------------------------------------------- @@ -82,6 +120,7 @@ class ClusteringStrategy: max_per_cluster: int, *, depth: int = 2, + root: str | None = None, ) -> list[list[str]]: """Group *files* by directory prefix. @@ -89,13 +128,14 @@ class ClusteringStrategy: files: File paths to partition. max_per_cluster: Maximum files per cluster. depth: Number of leading path components for the grouping key. + root: Optional root directory for relative path computation. Returns: Ordered list of clusters. """ buckets: dict[str, list[str]] = defaultdict(list) for f in sorted(files): - buckets[_directory_key(f, depth=depth)].append(f) + buckets[_directory_key(f, depth=depth, root=root)].append(f) clusters: list[list[str]] = [] for key in sorted(buckets): diff --git a/src/cleveragents/application/services/decomposition_service.py b/src/cleveragents/application/services/decomposition_service.py index 4e9a39392..d96e5ed8c 100644 --- a/src/cleveragents/application/services/decomposition_service.py +++ b/src/cleveragents/application/services/decomposition_service.py @@ -231,8 +231,10 @@ class DecompositionService: return depth # Cluster using directory strategy first, fall back to language + # Compute common root for relative path computation + common_root = _common_prefix(files) clusters = ClusteringStrategy.cluster_by_directory( - files, config.max_files_per_subplan + files, config.max_files_per_subplan, root=common_root ) strategy = ClusterStrategy.DIRECTORY