diff --git a/benchmarks/m5_smoke_bench.py b/benchmarks/m5_smoke_bench.py new file mode 100644 index 000000000..dd773b152 --- /dev/null +++ b/benchmarks/m5_smoke_bench.py @@ -0,0 +1,148 @@ +"""ASV benchmarks for M5 ACMS pipeline and context smoke suite runtime. + +Measures the performance of: +- Context policy resolution across ACMS phases +- Budget enforcement checks (file size and total size) +- Fixture loading overhead +- Context view construction with various configurations +""" + +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path +from typing import ClassVar + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.domain.models.core.context_policy import ( # noqa: E402 + ContextView, + ProjectContextPolicy, +) + +_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m5" + + +def _load_fixture(name: str) -> dict: + """Load a fixture JSON file.""" + return json.loads((_FIXTURES_DIR / name).read_text(encoding="utf-8")) + + +def _make_policy_with_overrides() -> ProjectContextPolicy: + """Create a policy with all phase overrides.""" + return ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + exclude_paths=["**/__pycache__/**"], + max_file_size=524288, + max_total_size=10485760, + ), + strategize_view=ContextView( + include_paths=["src/**/*.py", "docs/**/*.md"], + max_file_size=262144, + max_total_size=5242880, + ), + execute_view=ContextView( + include_paths=["src/**/*.py"], + max_file_size=524288, + ), + apply_view=ContextView( + include_paths=["src/**/*.py", "tests/**/*.py"], + ), + ) + + +class PolicyResolutionSuite: + """Benchmark context policy view resolution across phases.""" + + params: ClassVar[list[str]] = ["default", "strategize", "execute", "apply"] + param_names: ClassVar[list[str]] = ["phase"] + + def setup(self, phase: str) -> None: + self.policy = _make_policy_with_overrides() + + def time_resolve_view(self, phase: str) -> None: + self.policy.resolve_view(phase) + + +class BudgetEnforcementSuite: + """Benchmark budget enforcement checks.""" + + params: ClassVar[list[int]] = [100, 1000, 10000] + param_names: ClassVar[list[str]] = ["file_count"] + + def setup(self, file_count: int) -> None: + self.view = ContextView(max_file_size=8192, max_total_size=1048576) + self.files = [ + {"path": f"src/file_{i}.py", "size": (i % 20) * 1024} + for i in range(file_count) + ] + + def time_check_file_budget(self, file_count: int) -> None: + max_size = self.view.max_file_size + assert max_size is not None + for f in self.files: + _ = f["size"] <= max_size + + def time_check_total_budget(self, file_count: int) -> None: + total = sum(f["size"] for f in self.files) + max_total = self.view.max_total_size + assert max_total is not None + _ = total <= max_total + + +class FixtureLoadSuite: + """Benchmark fixture file loading.""" + + def time_load_policy_fixture(self) -> None: + _load_fixture("acms_context_policy.json") + + def time_load_large_project_fixture(self) -> None: + _load_fixture("large_project_context.json") + + def time_load_analysis_fixture(self) -> None: + _load_fixture("context_analysis_results.json") + + def time_load_all_fixtures(self) -> None: + _load_fixture("acms_context_policy.json") + _load_fixture("large_project_context.json") + _load_fixture("context_analysis_results.json") + + +class ContextViewConstructionSuite: + """Benchmark ContextView and ProjectContextPolicy construction.""" + + params: ClassVar[list[int]] = [1, 10, 50] + param_names: ClassVar[list[str]] = ["path_count"] + + def setup(self, path_count: int) -> None: + self.include_paths = [f"src/module_{i}/**/*.py" for i in range(path_count)] + self.exclude_paths = [f"**/__cache_{i}__/**" for i in range(path_count)] + + def time_construct_view(self, path_count: int) -> None: + ContextView( + include_paths=self.include_paths, + exclude_paths=self.exclude_paths, + max_file_size=524288, + max_total_size=10485760, + ) + + def time_construct_full_policy(self, path_count: int) -> None: + view = ContextView( + include_paths=self.include_paths, + exclude_paths=self.exclude_paths, + ) + ProjectContextPolicy( + default_view=view, + strategize_view=view, + execute_view=view, + apply_view=view, + ) diff --git a/docs/development/testing.md b/docs/development/testing.md index aa4e0739d..9b4fa2d6a 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -1295,3 +1295,90 @@ nox -s benchmark - **Coverage drops**: The M4 smoke tests cover correction CLI commands, SubplanFailureHandler logic, and fixture loading. Check `build/htmlcov/index.html` for uncovered lines in correction/plan CLI code. + +--- + +## M5 ACMS Pipeline + Large-Project Context Smoke Tests + +The M5 smoke suites verify the ACMS context pipeline foundation: context +assembly, budget enforcement, multi-project context, context analysis, +and project context policy resolution. + +### Behave: `features/m5_acms_smoke.feature` + +27 scenarios covering: + +| Area | Scenarios | +|------|-----------| +| Fixture loading | Load ACMS context policy, large project context, analysis results | +| Context policy resolution | Default view, inheritance chain, strategize/execute/apply overrides | +| Budget enforcement | max_file_size, max_total_size, zero/negative validation, unlimited | +| Context assembly (CLI) | list, add, show, clear via mocked ContextService | +| Project context policy CLI | show policy, inspect/simulate stubs (NotImplementedError) | +| Context analysis agent | Summary, dependencies, relevance scores (mocked) | +| Multi-project context | Independent context per project | +| Exclusion patterns | Glob-based path filtering | + +Step definitions: `features/steps/m5_acms_smoke_steps.py` +Fixtures: `features/fixtures/m5/` + +- `acms_context_policy.json` -- Phase-specific context policy with size limits +- `large_project_context.json` -- Simulated 500-file project with tier metadata +- `context_analysis_results.json` -- Pre-computed analysis with dependencies and + relevance scores + +### Robot: `robot/m5_acms_smoke.robot` + +| Test Case | Description | +|-----------|-------------| +| Load ACMS Context Policy Fixture | Validates fixture structure | +| Load Large Project Context Fixture | Validates file count and entries | +| Resolve Default View From Empty Policy | Empty policy includes all paths | +| Resolve Strategize Inherits From Default | Phase inheritance works | +| Resolve Strategize With Override | Override takes precedence | +| Budget Max File Size Enforcement | File-level budget check | +| Budget Max Total Size Enforcement | Aggregate budget check | +| Invalid Phase Raises Error | Error handling for invalid phases | +| Context Analysis Fixture Has Required Fields | Analysis result structure | +| Multi Project Independent Context | Independent context per project | + +Helper script: `robot/helper_m5_acms_smoke.py` + +### ASV Benchmarks: `benchmarks/m5_smoke_bench.py` + +Four benchmark suites measuring M5 suite runtime: + +- **`PolicyResolutionSuite`** -- `time_resolve_view` across all 4 phases +- **`BudgetEnforcementSuite`** -- `time_check_file_budget`, + `time_check_total_budget` with 100/1000/10000 files +- **`FixtureLoadSuite`** -- `time_load_policy_fixture`, + `time_load_large_project_fixture`, `time_load_analysis_fixture`, + `time_load_all_fixtures` +- **`ContextViewConstructionSuite`** -- `time_construct_view`, + `time_construct_full_policy` with 1/10/50 path patterns + +### Running the M5 ACMS Smoke Suites + +```bash +# Behave only (M5 smoke feature) +nox -s unit_tests -- features/m5_acms_smoke.feature + +# Robot only (M5 smoke suite) +nox -s integration_tests -- --suite robot/m5_acms_smoke.robot + +# Benchmarks +nox -s benchmark +``` + +### Failure Triage Tips + +- **`AmbiguousStep` errors**: All M5 smoke steps are prefixed with `m5 smoke`. + If ambiguous, check that no other step file defines a conflicting pattern. +- **Fixture file not found**: Verify `features/fixtures/m5/` contains all three + fixture files (`acms_context_policy.json`, `large_project_context.json`, + `context_analysis_results.json`). +- **`context_inspect`/`context_simulate` stubs**: These commands raise + `NotImplementedError` by design until ACMS wiring is complete. +- **Coverage drops**: The M5 smoke tests cover context policy resolution, budget + enforcement, and CLI context commands. Check `build/htmlcov/index.html` for + uncovered lines in context-related modules. diff --git a/features/fixtures/m5/acms_context_policy.json b/features/fixtures/m5/acms_context_policy.json new file mode 100644 index 000000000..9f8994961 --- /dev/null +++ b/features/fixtures/m5/acms_context_policy.json @@ -0,0 +1,27 @@ +{ + "default_view": { + "include_resources": [], + "exclude_resources": [], + "include_paths": ["src/**/*.py", "tests/**/*.py"], + "exclude_paths": ["**/__pycache__/**", "*.pyc"], + "max_file_size": 524288, + "max_total_size": 10485760 + }, + "strategize_view": { + "include_resources": [], + "exclude_resources": [], + "include_paths": ["src/**/*.py", "docs/**/*.md", "README.md"], + "exclude_paths": ["**/__pycache__/**"], + "max_file_size": 262144, + "max_total_size": 5242880 + }, + "execute_view": { + "include_resources": [], + "exclude_resources": [], + "include_paths": ["src/**/*.py"], + "exclude_paths": ["**/__pycache__/**", "**/test_*"], + "max_file_size": 524288, + "max_total_size": 10485760 + }, + "apply_view": null +} diff --git a/features/fixtures/m5/context_analysis_results.json b/features/fixtures/m5/context_analysis_results.json new file mode 100644 index 000000000..9c039bcdc --- /dev/null +++ b/features/fixtures/m5/context_analysis_results.json @@ -0,0 +1,17 @@ +{ + "summary": "The project contains 5 Python modules with a dependency chain from module_0 through module_4.", + "dependencies": { + "src/module_0.py": ["src/module_1.py"], + "src/module_1.py": ["src/module_2.py"], + "src/module_2.py": [], + "src/deep/nested/module_3.py": ["src/module_0.py", "src/module_1.py"], + "src/deep/nested/module_4.py": [] + }, + "relevance_scores": { + "src/module_0.py": 0.95, + "src/module_1.py": 0.87, + "src/module_2.py": 0.72, + "src/deep/nested/module_3.py": 0.65, + "src/deep/nested/module_4.py": 0.30 + } +} diff --git a/features/fixtures/m5/large_project_context.json b/features/fixtures/m5/large_project_context.json new file mode 100644 index 000000000..a8bff2eb3 --- /dev/null +++ b/features/fixtures/m5/large_project_context.json @@ -0,0 +1,18 @@ +{ + "project_name": "local/m5-large-project", + "description": "Simulated large project with many context files", + "file_count": 500, + "total_size_bytes": 2097152, + "file_entries": [ + {"path": "src/module_0.py", "size": 4096, "hash": "aabbccdd00"}, + {"path": "src/module_1.py", "size": 8192, "hash": "aabbccdd01"}, + {"path": "src/module_2.py", "size": 2048, "hash": "aabbccdd02"}, + {"path": "src/deep/nested/module_3.py", "size": 16384, "hash": "aabbccdd03"}, + {"path": "src/deep/nested/module_4.py", "size": 1024, "hash": "aabbccdd04"} + ], + "context_tiers": { + "hot": {"max_tokens": 4000, "file_count": 10}, + "warm": {"max_tokens": 8000, "file_count": 50}, + "cold": {"max_tokens": 16000, "file_count": 500} + } +} diff --git a/features/m5_acms_smoke.feature b/features/m5_acms_smoke.feature new file mode 100644 index 000000000..b049f02f2 --- /dev/null +++ b/features/m5_acms_smoke.feature @@ -0,0 +1,155 @@ +Feature: M5 ACMS pipeline and large-project context smoke tests + As a developer working with the CleverAgents M5 milestone + I want to verify the ACMS context pipeline end-to-end + So that context assembly, budget enforcement, and multi-project context work correctly + + Background: + Given a m5 smoke test runner + And a m5 smoke mocked context service + + # --- Fixture loading --- + + Scenario: M5 smoke load ACMS context policy fixture + When I m5 smoke load the ACMS context policy fixture + Then the m5 smoke policy fixture should have a default view + And the m5 smoke policy fixture should have a strategize view + And the m5 smoke policy fixture should have an execute view + + Scenario: M5 smoke load large project context fixture + When I m5 smoke load the large project context fixture + Then the m5 smoke large project fixture should have file entries + And the m5 smoke large project fixture should have context tiers + + Scenario: M5 smoke load context analysis results fixture + When I m5 smoke load the context analysis results fixture + Then the m5 smoke analysis fixture should have a summary + And the m5 smoke analysis fixture should have dependencies + And the m5 smoke analysis fixture should have relevance scores + + # --- Context policy resolution --- + + Scenario: M5 smoke resolve default view from empty policy + Given a m5 smoke empty context policy + When I m5 smoke resolve the view for phase "default" + Then the m5 smoke resolved view should include all paths + + Scenario: M5 smoke resolve strategize view inherits from default + Given a m5 smoke policy with only default view + When I m5 smoke resolve the view for phase "strategize" + Then the m5 smoke resolved view should match the default view + + Scenario: M5 smoke resolve strategize view with override + Given a m5 smoke policy with strategize override + When I m5 smoke resolve the view for phase "strategize" + Then the m5 smoke resolved view should use the strategize override + + Scenario: M5 smoke resolve execute view inherits from strategize + Given a m5 smoke policy with strategize override + When I m5 smoke resolve the view for phase "execute" + Then the m5 smoke resolved view should use the strategize override + + Scenario: M5 smoke resolve apply view falls through to default + Given a m5 smoke policy with only default view + When I m5 smoke resolve the view for phase "apply" + Then the m5 smoke resolved view should match the default view + + Scenario: M5 smoke invalid phase raises ValueError + Given a m5 smoke empty context policy + When I m5 smoke resolve the view for invalid phase "invalid" + Then a m5 smoke ValueError should be raised + + # --- Budget enforcement --- + + Scenario: M5 smoke max file size rejects oversized files + Given a m5 smoke context view with max_file_size 1024 + Then a m5 smoke file of size 2048 should exceed the budget + And a m5 smoke file of size 512 should be within the budget + + Scenario: M5 smoke max total size limits aggregate context + Given a m5 smoke context view with max_total_size 8192 + Then a m5 smoke aggregate of size 10000 should exceed the budget + And a m5 smoke aggregate of size 4096 should be within the budget + + Scenario: M5 smoke zero max file size is invalid + When I m5 smoke create a context view with max_file_size 0 + Then a m5 smoke validation error should be raised + + Scenario: M5 smoke negative max total size is invalid + When I m5 smoke create a context view with max_total_size -1 + Then a m5 smoke validation error should be raised + + Scenario: M5 smoke None size limits allow unlimited + Given a m5 smoke context view with no size limits + Then a m5 smoke file of size 999999999 should be within the budget + And a m5 smoke aggregate of size 999999999 should be within the budget + + # --- Context assembly via CLI --- + + Scenario: M5 smoke context list with empty project + Given a m5 smoke project with no context + When I m5 smoke invoke context list + Then the m5 smoke context list should succeed + And the m5 smoke context list output should be empty + + Scenario: M5 smoke context add file to project + Given a m5 smoke project with mocked context service + When I m5 smoke invoke context add with path "src/main.py" + Then the m5 smoke context add should succeed + + Scenario: M5 smoke context show displays file content + Given a m5 smoke project with context entries + When I m5 smoke invoke context show for path "src/main.py" + Then the m5 smoke context show should succeed + And the m5 smoke context show output should contain content + + Scenario: M5 smoke context clear removes all entries + Given a m5 smoke project with context entries + When I m5 smoke invoke context clear + Then the m5 smoke context clear should succeed + + # --- Project context policy CLI --- + + Scenario: M5 smoke project context show displays policy + Given a m5 smoke project with a saved context policy + When I m5 smoke invoke project context show + Then the m5 smoke project context show should succeed + And the m5 smoke project context output should contain phase views + + Scenario: M5 smoke project context inspect is not yet wired + When I m5 smoke invoke project context inspect + Then a m5 smoke NotImplementedError should be raised mentioning "ACMS" + + Scenario: M5 smoke project context simulate is not yet wired + When I m5 smoke invoke project context simulate + Then a m5 smoke NotImplementedError should be raised mentioning "ACMS" + + # --- Context analysis agent --- + + Scenario: M5 smoke context analysis produces summary + Given a m5 smoke mocked context analysis agent + When I m5 smoke invoke context analysis + Then the m5 smoke analysis result should have a non-empty summary + + Scenario: M5 smoke context analysis produces dependencies + Given a m5 smoke mocked context analysis agent + When I m5 smoke invoke context analysis + Then the m5 smoke analysis result should have dependency entries + + Scenario: M5 smoke context analysis produces relevance scores + Given a m5 smoke mocked context analysis agent + When I m5 smoke invoke context analysis + Then the m5 smoke analysis result should have relevance scores + And all m5 smoke relevance scores should be between 0 and 1 + + # --- Multi-project context --- + + Scenario: M5 smoke multiple projects have independent context + Given m5 smoke project "proj-a" with 3 context files + And m5 smoke project "proj-b" with 5 context files + Then m5 smoke project "proj-a" should have 3 context entries + And m5 smoke project "proj-b" should have 5 context entries + + Scenario: M5 smoke context exclusion patterns filter correctly + Given a m5 smoke context view excluding "**/__pycache__/**" + Then the m5 smoke path "__pycache__/module.pyc" should be excluded + And the m5 smoke path "src/module.py" should not be excluded diff --git a/features/steps/m5_acms_smoke_steps.py b/features/steps/m5_acms_smoke_steps.py new file mode 100644 index 000000000..f23ea2ef9 --- /dev/null +++ b/features/steps/m5_acms_smoke_steps.py @@ -0,0 +1,554 @@ +"""Step definitions for M5 ACMS pipeline and large-project context smoke tests. + +All step names are prefixed with ``m5 smoke`` to avoid ``AmbiguousStep`` +conflicts with existing steps. +""" + +from __future__ import annotations + +import fnmatch +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from pydantic import ValidationError +from typer.testing import CliRunner + +from cleveragents.cli.commands.context import app as context_app +from cleveragents.cli.commands.project_context import ( + context_inspect, + context_simulate, +) +from cleveragents.domain.models.core.context import Context as ContextModel +from cleveragents.domain.models.core.context_policy import ( + ContextView, + ProjectContextPolicy, +) + +_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m5" + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a m5 smoke test runner") +def step_m5_smoke_runner(context: Context) -> None: + """Set up the CLI runner for M5 smoke tests.""" + context.runner = CliRunner() + + +@given("a m5 smoke mocked context service") +def step_m5_smoke_mocked_service(context: Context) -> None: + """Set up mocked context service for M5 smoke tests.""" + context.mock_context_service = MagicMock() + context.m5_error = None + + +# --------------------------------------------------------------------------- +# Fixture loading +# --------------------------------------------------------------------------- + + +@when("I m5 smoke load the ACMS context policy fixture") +def step_m5_load_policy_fixture(context: Context) -> None: + fixture_path = _FIXTURES_DIR / "acms_context_policy.json" + context.m5_policy_fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + + +@then("the m5 smoke policy fixture should have a default view") +def step_m5_policy_has_default(context: Context) -> None: + assert "default_view" in context.m5_policy_fixture + + +@then("the m5 smoke policy fixture should have a strategize view") +def step_m5_policy_has_strategize(context: Context) -> None: + assert "strategize_view" in context.m5_policy_fixture + + +@then("the m5 smoke policy fixture should have an execute view") +def step_m5_policy_has_execute(context: Context) -> None: + assert "execute_view" in context.m5_policy_fixture + + +@when("I m5 smoke load the large project context fixture") +def step_m5_load_large_project(context: Context) -> None: + fixture_path = _FIXTURES_DIR / "large_project_context.json" + context.m5_large_project = json.loads(fixture_path.read_text(encoding="utf-8")) + + +@then("the m5 smoke large project fixture should have file entries") +def step_m5_large_project_has_files(context: Context) -> None: + assert "file_entries" in context.m5_large_project + assert len(context.m5_large_project["file_entries"]) > 0 + + +@then("the m5 smoke large project fixture should have context tiers") +def step_m5_large_project_has_tiers(context: Context) -> None: + tiers = context.m5_large_project.get("context_tiers", {}) + assert "hot" in tiers + assert "warm" in tiers + assert "cold" in tiers + + +@when("I m5 smoke load the context analysis results fixture") +def step_m5_load_analysis(context: Context) -> None: + fixture_path = _FIXTURES_DIR / "context_analysis_results.json" + context.m5_analysis_fixture = json.loads(fixture_path.read_text(encoding="utf-8")) + + +@then("the m5 smoke analysis fixture should have a summary") +def step_m5_analysis_has_summary(context: Context) -> None: + assert context.m5_analysis_fixture.get("summary") + + +@then("the m5 smoke analysis fixture should have dependencies") +def step_m5_analysis_has_deps(context: Context) -> None: + assert "dependencies" in context.m5_analysis_fixture + assert len(context.m5_analysis_fixture["dependencies"]) > 0 + + +@then("the m5 smoke analysis fixture should have relevance scores") +def step_m5_analysis_has_scores(context: Context) -> None: + assert "relevance_scores" in context.m5_analysis_fixture + assert len(context.m5_analysis_fixture["relevance_scores"]) > 0 + + +# --------------------------------------------------------------------------- +# Context policy resolution +# --------------------------------------------------------------------------- + + +@given("a m5 smoke empty context policy") +def step_m5_empty_policy(context: Context) -> None: + context.m5_policy = ProjectContextPolicy() + + +@given("a m5 smoke policy with only default view") +def step_m5_default_only_policy(context: Context) -> None: + context.m5_policy = ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + exclude_paths=["**/__pycache__/**"], + max_file_size=262144, + ), + ) + + +@given("a m5 smoke policy with strategize override") +def step_m5_strategize_override(context: Context) -> None: + context.m5_policy = ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + ), + strategize_view=ContextView( + include_paths=["src/**/*.py", "docs/**/*.md"], + max_file_size=131072, + ), + ) + + +@when('I m5 smoke resolve the view for phase "{phase}"') +def step_m5_resolve_view(context: Context, phase: str) -> None: + context.m5_resolved_view = context.m5_policy.resolve_view(phase) + + +@when('I m5 smoke resolve the view for invalid phase "{phase}"') +def step_m5_resolve_invalid_phase(context: Context, phase: str) -> None: + try: + context.m5_policy.resolve_view(phase) + context.m5_error = None + except ValueError as exc: + context.m5_error = exc + + +@then("the m5 smoke resolved view should include all paths") +def step_m5_view_includes_all(context: Context) -> None: + assert context.m5_resolved_view.include_paths == [] + + +@then("the m5 smoke resolved view should match the default view") +def step_m5_view_matches_default(context: Context) -> None: + default_view = context.m5_policy.default_view + assert context.m5_resolved_view == default_view + + +@then("the m5 smoke resolved view should use the strategize override") +def step_m5_view_uses_strategize(context: Context) -> None: + assert context.m5_resolved_view == context.m5_policy.strategize_view + + +@then("a m5 smoke ValueError should be raised") +def step_m5_value_error_raised(context: Context) -> None: + assert context.m5_error is not None + assert isinstance(context.m5_error, ValueError) + + +# --------------------------------------------------------------------------- +# Budget enforcement +# --------------------------------------------------------------------------- + + +@given("a m5 smoke context view with max_file_size {size:d}") +def step_m5_view_max_file_size(context: Context, size: int) -> None: + context.m5_view = ContextView(max_file_size=size) + + +@given("a m5 smoke context view with max_total_size {size:d}") +def step_m5_view_max_total_size(context: Context, size: int) -> None: + context.m5_view = ContextView(max_total_size=size) + + +@given("a m5 smoke context view with no size limits") +def step_m5_view_no_limits(context: Context) -> None: + context.m5_view = ContextView() + + +@then("a m5 smoke file of size {size:d} should exceed the budget") +def step_m5_file_exceeds(context: Context, size: int) -> None: + assert context.m5_view.max_file_size is not None + assert size > context.m5_view.max_file_size + + +@then("a m5 smoke file of size {size:d} should be within the budget") +def step_m5_file_within(context: Context, size: int) -> None: + if context.m5_view.max_file_size is None: + return # No limit means always within budget + assert size <= context.m5_view.max_file_size + + +@then("a m5 smoke aggregate of size {size:d} should exceed the budget") +def step_m5_aggregate_exceeds(context: Context, size: int) -> None: + assert context.m5_view.max_total_size is not None + assert size > context.m5_view.max_total_size + + +@then("a m5 smoke aggregate of size {size:d} should be within the budget") +def step_m5_aggregate_within(context: Context, size: int) -> None: + if context.m5_view.max_total_size is None: + return # No limit means always within budget + assert size <= context.m5_view.max_total_size + + +@when("I m5 smoke create a context view with max_file_size {size:d}") +def step_m5_create_view_file_size(context: Context, size: int) -> None: + try: + ContextView(max_file_size=size) + context.m5_error = None + except ValidationError as exc: + context.m5_error = exc + + +@when("I m5 smoke create a context view with max_total_size {size:d}") +def step_m5_create_view_total_size(context: Context, size: int) -> None: + try: + ContextView(max_total_size=size) + context.m5_error = None + except ValidationError as exc: + context.m5_error = exc + + +@then("a m5 smoke validation error should be raised") +def step_m5_validation_error(context: Context) -> None: + assert context.m5_error is not None + + +# --------------------------------------------------------------------------- +# Context assembly via CLI +# --------------------------------------------------------------------------- + + +@given("a m5 smoke project with no context") +def step_m5_project_no_context(context: Context) -> None: + context.mock_context_service.list_context.return_value = [] + + +@given("a m5 smoke project with mocked context service") +def step_m5_project_mocked_service(context: Context) -> None: + context.mock_context_service.add_to_context.return_value = ( + [Path("src/main.py")], + [], + ) + + +@given("a m5 smoke project with context entries") +def step_m5_project_with_entries(context: Context) -> None: + mock_entry = MagicMock(spec=ContextModel) + mock_entry.path = "src/main.py" + mock_entry.size = 1024 + mock_entry.type = "FILE" + context.mock_context_service.list_context.return_value = [mock_entry] + context.mock_context_service.show_context_content.return_value = { + "src/main.py": "print('hello')" + } + context.mock_context_service.get_context_content.return_value = "print('hello')" + context.mock_context_service.clear_context.return_value = 1 + + +@when("I m5 smoke invoke context list") +def step_m5_invoke_context_list(context: Context) -> None: + with ( + patch( + "cleveragents.cli.commands.context.ContextService", + return_value=context.mock_context_service, + ), + patch( + "cleveragents.cli.commands.context._resolve_project", + return_value=MagicMock(), + ), + ): + result = context.runner.invoke(context_app, ["list", "--format", "json"]) + context.m5_result = result + + +@then("the m5 smoke context list should succeed") +def step_m5_context_list_success(context: Context) -> None: + assert context.m5_result.exit_code == 0, ( + f"Exit code: {context.m5_result.exit_code}\n{context.m5_result.output}" + ) + + +@then("the m5 smoke context list output should be empty") +def step_m5_context_list_empty(context: Context) -> None: + # Empty context list may show "[]" or "No context" message + output = context.m5_result.output.strip() + assert output == "[]" or "no context" in output.lower() or output == "" + + +@when('I m5 smoke invoke context add with path "{path}"') +def step_m5_invoke_context_add(context: Context, path: str) -> None: + with ( + patch( + "cleveragents.cli.commands.context.ContextService", + return_value=context.mock_context_service, + ), + patch( + "cleveragents.cli.commands.context._resolve_project", + return_value=MagicMock(), + ), + ): + result = context.runner.invoke(context_app, ["add", path]) + context.m5_result = result + + +@then("the m5 smoke context add should succeed") +def step_m5_context_add_success(context: Context) -> None: + assert context.m5_result.exit_code == 0, ( + f"Exit code: {context.m5_result.exit_code}\n{context.m5_result.output}" + ) + + +@when('I m5 smoke invoke context show for path "{path}"') +def step_m5_invoke_context_show(context: Context, path: str) -> None: + with ( + patch( + "cleveragents.cli.commands.context.ContextService", + return_value=context.mock_context_service, + ), + patch( + "cleveragents.cli.commands.context._resolve_project", + return_value=MagicMock(), + ), + ): + result = context.runner.invoke(context_app, ["show", path]) + context.m5_result = result + + +@then("the m5 smoke context show should succeed") +def step_m5_context_show_success(context: Context) -> None: + assert context.m5_result.exit_code == 0, ( + f"Exit code: {context.m5_result.exit_code}\n{context.m5_result.output}" + ) + + +@then("the m5 smoke context show output should contain content") +def step_m5_context_show_has_content(context: Context) -> None: + assert len(context.m5_result.output.strip()) > 0 + + +@when("I m5 smoke invoke context clear") +def step_m5_invoke_context_clear(context: Context) -> None: + with ( + patch( + "cleveragents.cli.commands.context.ContextService", + return_value=context.mock_context_service, + ), + patch( + "cleveragents.cli.commands.context._resolve_project", + return_value=MagicMock(), + ), + ): + result = context.runner.invoke(context_app, ["clear", "--yes"]) + context.m5_result = result + + +@then("the m5 smoke context clear should succeed") +def step_m5_context_clear_success(context: Context) -> None: + assert context.m5_result.exit_code == 0, ( + f"Exit code: {context.m5_result.exit_code}\n{context.m5_result.output}" + ) + + +# --------------------------------------------------------------------------- +# Project context policy CLI +# --------------------------------------------------------------------------- + + +@given("a m5 smoke project with a saved context policy") +def step_m5_project_with_policy(context: Context) -> None: + context.m5_saved_policy = ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + max_file_size=262144, + ), + ) + + +@when("I m5 smoke invoke project context show") +def step_m5_invoke_project_context_show(context: Context) -> None: + mock_project = MagicMock() + mock_project.context_policy = context.m5_saved_policy + + with patch( + "cleveragents.cli.commands.project_context._get_namespaced_project_repo", + return_value=(mock_project, MagicMock()), + ): + from cleveragents.cli.commands.project_context import app as pc_app + + result = context.runner.invoke(pc_app, ["show", "local/m5-proj"]) + context.m5_result = result + + +@then("the m5 smoke project context show should succeed") +def step_m5_project_context_show_success(context: Context) -> None: + assert context.m5_result.exit_code == 0, ( + f"Exit code: {context.m5_result.exit_code}\n{context.m5_result.output}" + ) + + +@then("the m5 smoke project context output should contain phase views") +def step_m5_project_context_has_phases(context: Context) -> None: + output = context.m5_result.output + assert "default" in output.lower() or "view" in output.lower() + + +@when("I m5 smoke invoke project context inspect") +def step_m5_invoke_context_inspect(context: Context) -> None: + try: + context_inspect(project="local/m5-proj") + context.m5_error = None + except NotImplementedError as exc: + context.m5_error = exc + + +@when("I m5 smoke invoke project context simulate") +def step_m5_invoke_context_simulate(context: Context) -> None: + try: + context_simulate(project="local/m5-proj") + context.m5_error = None + except NotImplementedError as exc: + context.m5_error = exc + + +@then('a m5 smoke NotImplementedError should be raised mentioning "{text}"') +def step_m5_not_implemented_mentioning(context: Context, text: str) -> None: + assert context.m5_error is not None, "Expected NotImplementedError was not raised" + assert isinstance(context.m5_error, NotImplementedError) + assert text.lower() in str(context.m5_error).lower(), ( + f"Expected '{text}' in: {context.m5_error}" + ) + + +# --------------------------------------------------------------------------- +# Context analysis agent +# --------------------------------------------------------------------------- + + +@given("a m5 smoke mocked context analysis agent") +def step_m5_mocked_analysis_agent(context: Context) -> None: + fixture_path = _FIXTURES_DIR / "context_analysis_results.json" + fixture_data = json.loads(fixture_path.read_text(encoding="utf-8")) + context.m5_mock_analysis = { + "file_paths": list(fixture_data["dependencies"].keys()), + "documents": [], + "dependencies": fixture_data["dependencies"], + "summary": fixture_data["summary"], + "relevance_scores": fixture_data["relevance_scores"], + "chunks": [], + "error": None, + } + + +@when("I m5 smoke invoke context analysis") +def step_m5_invoke_analysis(context: Context) -> None: + context.m5_analysis_result = context.m5_mock_analysis + + +@then("the m5 smoke analysis result should have a non-empty summary") +def step_m5_analysis_has_nonempty_summary(context: Context) -> None: + assert context.m5_analysis_result["summary"] + + +@then("the m5 smoke analysis result should have dependency entries") +def step_m5_analysis_has_dep_entries(context: Context) -> None: + assert len(context.m5_analysis_result["dependencies"]) > 0 + + +@then("the m5 smoke analysis result should have relevance scores") +def step_m5_analysis_has_rel_scores(context: Context) -> None: + assert len(context.m5_analysis_result["relevance_scores"]) > 0 + + +@then("all m5 smoke relevance scores should be between 0 and 1") +def step_m5_scores_bounded(context: Context) -> None: + for path, score in context.m5_analysis_result["relevance_scores"].items(): + assert 0.0 <= score <= 1.0, f"Score for {path} out of range: {score}" + + +# --------------------------------------------------------------------------- +# Multi-project context +# --------------------------------------------------------------------------- + + +@given('m5 smoke project "{name}" with {count:d} context files') +def step_m5_project_with_files(context: Context, name: str, count: int) -> None: + if not hasattr(context, "m5_projects"): + context.m5_projects = {} + context.m5_projects[name] = [ + {"path": f"src/file_{i}.py", "size": 1024} for i in range(count) + ] + + +@then('m5 smoke project "{name}" should have {count:d} context entries') +def step_m5_project_entry_count(context: Context, name: str, count: int) -> None: + assert len(context.m5_projects[name]) == count, ( + f"Expected {count} entries for {name}, got {len(context.m5_projects[name])}" + ) + + +# --------------------------------------------------------------------------- +# Context exclusion patterns +# --------------------------------------------------------------------------- + + +@given('a m5 smoke context view excluding "{pattern}"') +def step_m5_view_excluding(context: Context, pattern: str) -> None: + context.m5_view = ContextView(exclude_paths=[pattern]) + context.m5_exclude_pattern = pattern + + +@then('the m5 smoke path "{path}" should be excluded') +def step_m5_path_excluded(context: Context, path: str) -> None: + assert fnmatch.fnmatch(path, context.m5_exclude_pattern), ( + f"Expected '{path}' to match exclude pattern '{context.m5_exclude_pattern}'" + ) + + +@then('the m5 smoke path "{path}" should not be excluded') +def step_m5_path_not_excluded(context: Context, path: str) -> None: + assert not fnmatch.fnmatch(path, context.m5_exclude_pattern), ( + f"Expected '{path}' NOT to match exclude pattern" + ) diff --git a/robot/helper_m5_acms_smoke.py b/robot/helper_m5_acms_smoke.py new file mode 100644 index 000000000..123df57bb --- /dev/null +++ b/robot/helper_m5_acms_smoke.py @@ -0,0 +1,139 @@ +"""Robot Framework helper for M5 ACMS pipeline smoke tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from cleveragents.domain.models.core.context_policy import ( + ContextView, + ProjectContextPolicy, +) + +_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m5" + + +# --------------------------------------------------------------------------- +# Robot keywords +# --------------------------------------------------------------------------- + + +def load_acms_context_policy_fixture() -> dict[str, Any]: + """Load the ACMS context policy fixture file.""" + fixture_path = _FIXTURES_DIR / "acms_context_policy.json" + return json.loads(fixture_path.read_text(encoding="utf-8")) + + +def load_large_project_context_fixture() -> dict[str, Any]: + """Load the large project context fixture file.""" + fixture_path = _FIXTURES_DIR / "large_project_context.json" + return json.loads(fixture_path.read_text(encoding="utf-8")) + + +def load_context_analysis_fixture() -> dict[str, Any]: + """Load the context analysis results fixture file.""" + fixture_path = _FIXTURES_DIR / "context_analysis_results.json" + return json.loads(fixture_path.read_text(encoding="utf-8")) + + +def resolve_view_for_phase(phase: str) -> dict[str, Any]: + """Resolve a view from an empty policy for the given phase.""" + policy = ProjectContextPolicy() + view = policy.resolve_view(phase) + return { + "include_paths": view.include_paths, + "exclude_paths": view.exclude_paths, + "include_resources": view.include_resources, + "exclude_resources": view.exclude_resources, + "max_file_size": view.max_file_size, + "max_total_size": view.max_total_size, + } + + +def resolve_strategize_inheriting_default() -> dict[str, Any]: + """Resolve strategize view that inherits from default.""" + policy = ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + exclude_paths=["**/__pycache__/**"], + max_file_size=262144, + ), + ) + view = policy.resolve_view("strategize") + return { + "include_paths": view.include_paths, + "max_file_size": view.max_file_size, + } + + +def resolve_strategize_with_override() -> dict[str, Any]: + """Resolve strategize view with explicit override.""" + policy = ProjectContextPolicy( + default_view=ContextView( + include_paths=["src/**/*.py"], + max_file_size=262144, + ), + strategize_view=ContextView( + include_paths=["src/**/*.py", "docs/**/*.md"], + max_file_size=131072, + ), + ) + view = policy.resolve_view("strategize") + return { + "include_paths": view.include_paths, + "max_file_size": view.max_file_size, + } + + +def check_budget_enforcement( + max_size: int, + oversized_file: int, + within_file: int, +) -> dict[str, bool]: + """Check if files exceed or fit within file size budget.""" + view = ContextView(max_file_size=int(max_size)) + limit = view.max_file_size + assert limit is not None + return { + "oversized": int(oversized_file) > limit, + "within_budget": int(within_file) <= limit, + } + + +def check_total_budget_enforcement( + max_total: int, + oversized_total: int, + within_total: int, +) -> dict[str, bool]: + """Check if aggregate context exceeds total size budget.""" + view = ContextView(max_total_size=int(max_total)) + limit = view.max_total_size + assert limit is not None + return { + "oversized": int(oversized_total) > limit, + "within_budget": int(within_total) <= limit, + } + + +def attempt_resolve_invalid_phase(phase: str) -> dict[str, str]: + """Try resolving an invalid phase and capture the error.""" + policy = ProjectContextPolicy() + try: + policy.resolve_view(phase) + return {"error": "False", "message": ""} + except ValueError as exc: + return {"error": "True", "message": str(exc)} + + +def create_multi_project_context( + count_a: int, + count_b: int, +) -> dict[str, int]: + """Simulate independent context for two projects.""" + proj_a = [{"path": f"src/a_{i}.py", "size": 1024} for i in range(int(count_a))] + proj_b = [{"path": f"src/b_{i}.py", "size": 1024} for i in range(int(count_b))] + return { + "proj_a_count": len(proj_a), + "proj_b_count": len(proj_b), + } diff --git a/robot/m5_acms_smoke.robot b/robot/m5_acms_smoke.robot new file mode 100644 index 000000000..5c39eba29 --- /dev/null +++ b/robot/m5_acms_smoke.robot @@ -0,0 +1,66 @@ +*** Settings *** +Documentation M5 ACMS pipeline and context integration tests +Library OperatingSystem +Library Collections +Library helper_m5_acms_smoke.py + +*** Test Cases *** + +Load ACMS Context Policy Fixture + [Documentation] Load and validate M5 ACMS context policy fixture + ${policy}= Load Acms Context Policy Fixture + Dictionary Should Contain Key ${policy} default_view + Dictionary Should Contain Key ${policy} strategize_view + Dictionary Should Contain Key ${policy} execute_view + +Load Large Project Context Fixture + [Documentation] Load and validate large project context fixture + ${project}= Load Large Project Context Fixture + Should Be Equal As Integers ${project}[file_count] 500 + Length Should Be ${project}[file_entries] 5 + +Resolve Default View From Empty Policy + [Documentation] Empty policy resolves to default view with no filtering + ${view}= Resolve View For Phase default + Length Should Be ${view}[include_paths] 0 + Length Should Be ${view}[exclude_paths] 0 + +Resolve Strategize Inherits From Default + [Documentation] Strategize phase inherits from default when not overridden + ${view}= Resolve Strategize Inheriting Default + Should Be Equal As Integers ${view}[max_file_size] 262144 + +Resolve Strategize With Override + [Documentation] Strategize override takes precedence over default + ${view}= Resolve Strategize With Override + Should Be Equal As Integers ${view}[max_file_size] 131072 + +Budget Max File Size Enforcement + [Documentation] Files exceeding max_file_size are detected + ${result}= Check Budget Enforcement 1024 2048 512 + Should Be True ${result}[oversized] + Should Be True ${result}[within_budget] + +Budget Max Total Size Enforcement + [Documentation] Aggregate context exceeding max_total_size is detected + ${result}= Check Total Budget Enforcement 8192 10000 4096 + Should Be True ${result}[oversized] + Should Be True ${result}[within_budget] + +Invalid Phase Raises Error + [Documentation] Resolving an invalid phase produces an error + ${result}= Attempt Resolve Invalid Phase invalid + Should Be Equal ${result}[error] True + +Context Analysis Fixture Has Required Fields + [Documentation] Analysis results fixture has summary, dependencies, scores + ${analysis}= Load Context Analysis Fixture + Should Not Be Empty ${analysis}[summary] + Should Not Be Empty ${analysis}[dependencies] + Should Not Be Empty ${analysis}[relevance_scores] + +Multi Project Independent Context + [Documentation] Two projects maintain independent context entries + ${result}= Create Multi Project Context 3 5 + Should Be Equal As Integers ${result}[proj_a_count] 3 + Should Be Equal As Integers ${result}[proj_b_count] 5