Files
cleveragents-core/robot/helper_large_project_decompose.py
freemo 4232907ab9
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 18s
CI / build (pull_request) Successful in 23s
CI / typecheck (pull_request) Successful in 34s
CI / security (pull_request) Successful in 39s
CI / unit_tests (pull_request) Successful in 3m17s
CI / docker (pull_request) Successful in 40s
CI / integration_tests (pull_request) Successful in 4m3s
CI / coverage (pull_request) Successful in 4m9s
CI / lint (push) Successful in 12s
CI / quality (push) Successful in 16s
CI / build (push) Successful in 15s
CI / security (push) Successful in 29s
CI / typecheck (push) Successful in 33s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / integration_tests (push) Successful in 2m48s
CI / docker (push) Successful in 38s
CI / coverage (push) Successful in 4m16s
CI / benchmark-publish (push) Successful in 16m55s
CI / benchmark-regression (pull_request) Successful in 30m14s
feat(plan): add large-project decomposition and dependency closure
Add hierarchical decomposition with 4+ levels and bounded context per
subplan. Implement decomposition heuristics (max_files_per_subplan,
max_tokens_per_subplan, language/dir clustering). Add dependency closure
computation for large graphs and DAG execution ordering. Add bounded
dependency closure with cutoff thresholds and memoization for 10K+
files. Record decomposition decisions in DecisionService
(strategy_choice + subplan_spawn entries).

New modules:
- decomposition_models.py: DecompositionConfig, DecompositionNode,
  DecompositionResult, DependencyEdge, DependencyGraph
- decomposition_clustering.py: ClusteringStrategy with directory,
  language, and size clustering plus deterministic sort
- decomposition_graph.py: DependencyClosureComputer with bounded
  closure, topological sort, cycle detection, and memoization
- decomposition_service.py: DecompositionService orchestrating
  hierarchy building and decision recording

Settings: planner_max_depth, planner_max_files_per_subplan,
planner_max_tokens_per_subplan, planner_min_files_per_subplan

Closes #205
2026-03-03 21:44:20 +00:00

168 lines
5.4 KiB
Python

"""Helper script for large-project decomposition Robot Framework tests.
Usage:
python helper_large_project_decompose.py below-threshold
python helper_large_project_decompose.py multi-level
python helper_large_project_decompose.py closure
python helper_large_project_decompose.py topo-sort
python helper_large_project_decompose.py cycle-detect
python helper_large_project_decompose.py config-validation
"""
from __future__ import annotations
import os
import sys
import tempfile
from pathlib import Path
# Ensure source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.application.services.decomposition_graph import ( # noqa: E402
DependencyClosureComputer,
)
from cleveragents.application.services.decomposition_models import ( # noqa: E402
DecompositionConfig,
DependencyEdge,
DependencyGraph,
DependencyType,
)
from cleveragents.application.services.decomposition_service import ( # noqa: E402
DecompositionService,
)
def _make_files(tmpdir: str, count: int, levels: int = 1) -> list[str]:
"""Create *count* empty files spread across *levels* directories."""
paths: list[str] = []
for i in range(count):
level = i % levels
parts = [f"level{d}" for d in range(level + 1)]
dirpath = os.path.join(tmpdir, *parts)
os.makedirs(dirpath, exist_ok=True)
fpath = os.path.join(dirpath, f"file_{i:05d}.py")
with open(fpath, "w") as fh:
fh.write("x" * 200)
paths.append(fpath)
return sorted(paths)
def cmd_below_threshold() -> None:
"""Verify decomposition is skipped below threshold."""
tmpdir = tempfile.mkdtemp(prefix="decompose-")
files = _make_files(tmpdir, 5, 1)
svc = DecompositionService()
result = svc.decompose(files, DecompositionConfig(min_files_per_subplan=10))
assert result.metrics.get("skipped") == 1
assert len(result.nodes) == 1
print("decompose-below-threshold-ok")
def cmd_multi_level() -> None:
"""Verify multi-level decomposition produces multiple nodes."""
tmpdir = tempfile.mkdtemp(prefix="decompose-")
files = _make_files(tmpdir, 500, 5)
svc = DecompositionService()
result = svc.decompose(files, DecompositionConfig(max_files_per_subplan=100))
assert len(result.nodes) > 1
assert result.max_depth_reached >= 1
print(f"decompose-multi-level-ok nodes={len(result.nodes)}")
def cmd_closure() -> None:
"""Verify transitive closure computation."""
g = DependencyGraph()
for n in ("A", "B", "C", "D"):
g.add_node(n)
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
g.add_edge(DependencyEdge("B", "C", DependencyType.IMPORT))
g.add_edge(DependencyEdge("C", "D", DependencyType.IMPORT))
computer = DependencyClosureComputer()
closure = computer.compute_closure(g, ["A"])
assert {"A", "B", "C", "D"} <= closure
print(f"decompose-closure-ok size={len(closure)}")
def cmd_topo_sort() -> None:
"""Verify topological sort produces valid order."""
g = DependencyGraph()
for n in ("A", "B", "C", "D"):
g.add_node(n)
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
g.add_edge(DependencyEdge("A", "C", DependencyType.IMPORT))
g.add_edge(DependencyEdge("B", "D", DependencyType.IMPORT))
g.add_edge(DependencyEdge("C", "D", DependencyType.IMPORT))
computer = DependencyClosureComputer()
order = computer.topological_sort(g)
assert order.index("A") < order.index("B")
assert order.index("A") < order.index("C")
assert order.index("B") < order.index("D")
assert order.index("C") < order.index("D")
print(f"decompose-topo-sort-ok order={order}")
def cmd_cycle_detect() -> None:
"""Verify cycle detection finds a cycle."""
g = DependencyGraph()
for n in ("A", "B", "C"):
g.add_node(n)
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
g.add_edge(DependencyEdge("B", "C", DependencyType.IMPORT))
g.add_edge(DependencyEdge("C", "A", DependencyType.IMPORT))
computer = DependencyClosureComputer()
cycles = computer.detect_cycles(g)
assert len(cycles) >= 1
print(f"decompose-cycle-detect-ok cycles={len(cycles)}")
def cmd_config_validation() -> None:
"""Verify invalid config raises ValueError."""
for bad_kwargs in (
{"max_depth": 0},
{"max_files_per_subplan": 0},
{"max_tokens_per_subplan": 0},
{"min_files_per_subplan": 0},
):
try:
DecompositionConfig(**bad_kwargs) # type: ignore[arg-type]
print(f"FAIL: {bad_kwargs} did not raise")
sys.exit(1)
except ValueError:
pass
# Valid config should work
DecompositionConfig()
print("decompose-config-ok")
def main() -> None:
"""Dispatch subcommand."""
if len(sys.argv) < 2:
print("Usage: helper_large_project_decompose.py <command>")
sys.exit(1)
cmd = sys.argv[1]
dispatch = {
"below-threshold": cmd_below_threshold,
"multi-level": cmd_multi_level,
"closure": cmd_closure,
"topo-sort": cmd_topo_sort,
"cycle-detect": cmd_cycle_detect,
"config-validation": cmd_config_validation,
}
func = dispatch.get(cmd)
if func is None:
print(f"Unknown command: {cmd}")
sys.exit(1)
func()
if __name__ == "__main__":
main()