4232907ab9
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 18s
CI / build (pull_request) Successful in 23s
CI / typecheck (pull_request) Successful in 34s
CI / security (pull_request) Successful in 39s
CI / unit_tests (pull_request) Successful in 3m17s
CI / docker (pull_request) Successful in 40s
CI / integration_tests (pull_request) Successful in 4m3s
CI / coverage (pull_request) Successful in 4m9s
CI / lint (push) Successful in 12s
CI / quality (push) Successful in 16s
CI / build (push) Successful in 15s
CI / security (push) Successful in 29s
CI / typecheck (push) Successful in 33s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m23s
CI / integration_tests (push) Successful in 2m48s
CI / docker (push) Successful in 38s
CI / coverage (push) Successful in 4m16s
CI / benchmark-publish (push) Successful in 16m55s
CI / benchmark-regression (pull_request) Successful in 30m14s
Add hierarchical decomposition with 4+ levels and bounded context per subplan. Implement decomposition heuristics (max_files_per_subplan, max_tokens_per_subplan, language/dir clustering). Add dependency closure computation for large graphs and DAG execution ordering. Add bounded dependency closure with cutoff thresholds and memoization for 10K+ files. Record decomposition decisions in DecisionService (strategy_choice + subplan_spawn entries). New modules: - decomposition_models.py: DecompositionConfig, DecompositionNode, DecompositionResult, DependencyEdge, DependencyGraph - decomposition_clustering.py: ClusteringStrategy with directory, language, and size clustering plus deterministic sort - decomposition_graph.py: DependencyClosureComputer with bounded closure, topological sort, cycle detection, and memoization - decomposition_service.py: DecompositionService orchestrating hierarchy building and decision recording Settings: planner_max_depth, planner_max_files_per_subplan, planner_max_tokens_per_subplan, planner_min_files_per_subplan Closes #205
168 lines
5.4 KiB
Python
168 lines
5.4 KiB
Python
"""Helper script for large-project decomposition Robot Framework tests.
|
|
|
|
Usage:
|
|
python helper_large_project_decompose.py below-threshold
|
|
python helper_large_project_decompose.py multi-level
|
|
python helper_large_project_decompose.py closure
|
|
python helper_large_project_decompose.py topo-sort
|
|
python helper_large_project_decompose.py cycle-detect
|
|
python helper_large_project_decompose.py config-validation
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# Ensure source tree is importable
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
from cleveragents.application.services.decomposition_graph import ( # noqa: E402
|
|
DependencyClosureComputer,
|
|
)
|
|
from cleveragents.application.services.decomposition_models import ( # noqa: E402
|
|
DecompositionConfig,
|
|
DependencyEdge,
|
|
DependencyGraph,
|
|
DependencyType,
|
|
)
|
|
from cleveragents.application.services.decomposition_service import ( # noqa: E402
|
|
DecompositionService,
|
|
)
|
|
|
|
|
|
def _make_files(tmpdir: str, count: int, levels: int = 1) -> list[str]:
|
|
"""Create *count* empty files spread across *levels* directories."""
|
|
paths: list[str] = []
|
|
for i in range(count):
|
|
level = i % levels
|
|
parts = [f"level{d}" for d in range(level + 1)]
|
|
dirpath = os.path.join(tmpdir, *parts)
|
|
os.makedirs(dirpath, exist_ok=True)
|
|
fpath = os.path.join(dirpath, f"file_{i:05d}.py")
|
|
with open(fpath, "w") as fh:
|
|
fh.write("x" * 200)
|
|
paths.append(fpath)
|
|
return sorted(paths)
|
|
|
|
|
|
def cmd_below_threshold() -> None:
|
|
"""Verify decomposition is skipped below threshold."""
|
|
tmpdir = tempfile.mkdtemp(prefix="decompose-")
|
|
files = _make_files(tmpdir, 5, 1)
|
|
svc = DecompositionService()
|
|
result = svc.decompose(files, DecompositionConfig(min_files_per_subplan=10))
|
|
assert result.metrics.get("skipped") == 1
|
|
assert len(result.nodes) == 1
|
|
print("decompose-below-threshold-ok")
|
|
|
|
|
|
def cmd_multi_level() -> None:
|
|
"""Verify multi-level decomposition produces multiple nodes."""
|
|
tmpdir = tempfile.mkdtemp(prefix="decompose-")
|
|
files = _make_files(tmpdir, 500, 5)
|
|
svc = DecompositionService()
|
|
result = svc.decompose(files, DecompositionConfig(max_files_per_subplan=100))
|
|
assert len(result.nodes) > 1
|
|
assert result.max_depth_reached >= 1
|
|
print(f"decompose-multi-level-ok nodes={len(result.nodes)}")
|
|
|
|
|
|
def cmd_closure() -> None:
|
|
"""Verify transitive closure computation."""
|
|
g = DependencyGraph()
|
|
for n in ("A", "B", "C", "D"):
|
|
g.add_node(n)
|
|
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("B", "C", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("C", "D", DependencyType.IMPORT))
|
|
|
|
computer = DependencyClosureComputer()
|
|
closure = computer.compute_closure(g, ["A"])
|
|
assert {"A", "B", "C", "D"} <= closure
|
|
print(f"decompose-closure-ok size={len(closure)}")
|
|
|
|
|
|
def cmd_topo_sort() -> None:
|
|
"""Verify topological sort produces valid order."""
|
|
g = DependencyGraph()
|
|
for n in ("A", "B", "C", "D"):
|
|
g.add_node(n)
|
|
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("A", "C", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("B", "D", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("C", "D", DependencyType.IMPORT))
|
|
|
|
computer = DependencyClosureComputer()
|
|
order = computer.topological_sort(g)
|
|
assert order.index("A") < order.index("B")
|
|
assert order.index("A") < order.index("C")
|
|
assert order.index("B") < order.index("D")
|
|
assert order.index("C") < order.index("D")
|
|
print(f"decompose-topo-sort-ok order={order}")
|
|
|
|
|
|
def cmd_cycle_detect() -> None:
|
|
"""Verify cycle detection finds a cycle."""
|
|
g = DependencyGraph()
|
|
for n in ("A", "B", "C"):
|
|
g.add_node(n)
|
|
g.add_edge(DependencyEdge("A", "B", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("B", "C", DependencyType.IMPORT))
|
|
g.add_edge(DependencyEdge("C", "A", DependencyType.IMPORT))
|
|
|
|
computer = DependencyClosureComputer()
|
|
cycles = computer.detect_cycles(g)
|
|
assert len(cycles) >= 1
|
|
print(f"decompose-cycle-detect-ok cycles={len(cycles)}")
|
|
|
|
|
|
def cmd_config_validation() -> None:
|
|
"""Verify invalid config raises ValueError."""
|
|
for bad_kwargs in (
|
|
{"max_depth": 0},
|
|
{"max_files_per_subplan": 0},
|
|
{"max_tokens_per_subplan": 0},
|
|
{"min_files_per_subplan": 0},
|
|
):
|
|
try:
|
|
DecompositionConfig(**bad_kwargs) # type: ignore[arg-type]
|
|
print(f"FAIL: {bad_kwargs} did not raise")
|
|
sys.exit(1)
|
|
except ValueError:
|
|
pass
|
|
|
|
# Valid config should work
|
|
DecompositionConfig()
|
|
print("decompose-config-ok")
|
|
|
|
|
|
def main() -> None:
|
|
"""Dispatch subcommand."""
|
|
if len(sys.argv) < 2:
|
|
print("Usage: helper_large_project_decompose.py <command>")
|
|
sys.exit(1)
|
|
|
|
cmd = sys.argv[1]
|
|
dispatch = {
|
|
"below-threshold": cmd_below_threshold,
|
|
"multi-level": cmd_multi_level,
|
|
"closure": cmd_closure,
|
|
"topo-sort": cmd_topo_sort,
|
|
"cycle-detect": cmd_cycle_detect,
|
|
"config-validation": cmd_config_validation,
|
|
}
|
|
func = dispatch.get(cmd)
|
|
if func is None:
|
|
print(f"Unknown command: {cmd}")
|
|
sys.exit(1)
|
|
func()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|