test(e2e): add M3 decision + validation suites
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / build (pull_request) Successful in 20s
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m5s
CI / integration_tests (pull_request) Successful in 4m4s
CI / benchmark-regression (pull_request) Successful in 22m21s
CI / unit_tests (pull_request) Failing after 34m7s
CI / docker (pull_request) Has been skipped
CI / coverage (pull_request) Has been cancelled
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / build (pull_request) Successful in 20s
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m5s
CI / integration_tests (pull_request) Successful in 4m4s
CI / benchmark-regression (pull_request) Successful in 22m21s
CI / unit_tests (pull_request) Failing after 34m7s
CI / docker (pull_request) Has been skipped
CI / coverage (pull_request) Has been cancelled
This commit is contained in:
@@ -0,0 +1,255 @@
|
||||
"""ASV benchmarks for M3 decision tree, validation, and invariant CLI overhead.
|
||||
|
||||
Measures the performance of:
|
||||
- Invariant add / list / remove CLI commands
|
||||
- Validation add / attach / detach CLI commands
|
||||
- Plan correct (dry-run) CLI command
|
||||
- Decision tree fixture loading
|
||||
- Invariant merge precedence computation
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.invariant import app as invariant_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
|
||||
from cleveragents.domain.models.core.correction import ( # noqa: E402
|
||||
CorrectionMode,
|
||||
CorrectionStatus,
|
||||
)
|
||||
from cleveragents.domain.models.core.invariant import ( # noqa: E402
|
||||
Invariant,
|
||||
InvariantScope,
|
||||
)
|
||||
|
||||
_runner = CliRunner()
|
||||
_PLAN_ULID = "01M3SM0KE00000000000000001"
|
||||
_DECISION_ULID = "01M3DEC1S10N0000000000001"
|
||||
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
|
||||
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
|
||||
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m3"
|
||||
|
||||
|
||||
def _mock_invariant(
|
||||
*,
|
||||
text: str = "Never delete production data",
|
||||
scope: InvariantScope = InvariantScope.GLOBAL,
|
||||
source_name: str = "system",
|
||||
) -> Invariant:
|
||||
return Invariant(
|
||||
id=_INVARIANT_ULID,
|
||||
text=text,
|
||||
scope=scope,
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
|
||||
class M3InvariantAddSuite:
|
||||
"""Benchmark invariant add CLI command."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.add_invariant.return_value = _mock_invariant()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_invariant_add_global(self) -> None:
|
||||
"""Benchmark invariant add with global scope."""
|
||||
_runner.invoke(
|
||||
invariant_app,
|
||||
["add", "--global", "Never delete production data", "--format", "plain"],
|
||||
)
|
||||
|
||||
def time_invariant_add_project(self) -> None:
|
||||
"""Benchmark invariant add with project scope."""
|
||||
_runner.invoke(
|
||||
invariant_app,
|
||||
[
|
||||
"add",
|
||||
"--project",
|
||||
"local/m3-smoke-proj",
|
||||
"All API changes need tests",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class M3InvariantListSuite:
|
||||
"""Benchmark invariant list CLI command."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.list_invariants.return_value = [
|
||||
_mock_invariant(),
|
||||
_mock_invariant(
|
||||
text="All API changes need tests",
|
||||
scope=InvariantScope.PROJECT,
|
||||
source_name="local/m3-smoke-proj",
|
||||
),
|
||||
]
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_invariant_list(self) -> None:
|
||||
"""Benchmark invariant list command."""
|
||||
_runner.invoke(invariant_app, ["list", "--format", "plain"])
|
||||
|
||||
def time_invariant_list_with_project(self) -> None:
|
||||
"""Benchmark invariant list with project filter."""
|
||||
_runner.invoke(
|
||||
invariant_app,
|
||||
["list", "--project", "local/m3-smoke-proj", "--format", "plain"],
|
||||
)
|
||||
|
||||
|
||||
class M3ValidationAddSuite:
|
||||
"""Benchmark validation add CLI command."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
mock_validation = MagicMock()
|
||||
mock_validation.name = "local/coverage-check"
|
||||
mock_validation.as_cli_dict.return_value = {
|
||||
"name": "local/coverage-check",
|
||||
"description": "Check code coverage meets threshold",
|
||||
"source": "custom",
|
||||
"mode": "required",
|
||||
}
|
||||
self._mock_service.register_tool.return_value = mock_validation
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
config_content = (
|
||||
"name: local/coverage-check\n"
|
||||
"description: Check code coverage meets threshold\n"
|
||||
"source: custom\n"
|
||||
"mode: required\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': inputs['coverage'] >= 80}\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
self._config_path = tmp.name
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_validation_add(self) -> None:
|
||||
"""Benchmark validation add from YAML config."""
|
||||
_runner.invoke(
|
||||
validation_app,
|
||||
["add", "--config", self._config_path, "--format", "plain"],
|
||||
)
|
||||
|
||||
|
||||
class M3PlanCorrectSuite:
|
||||
"""Benchmark plan correct (dry-run) CLI command."""
|
||||
|
||||
def setup(self) -> None:
|
||||
mock_correction_svc = MagicMock()
|
||||
mock_request = MagicMock()
|
||||
mock_request.correction_id = _CORRECTION_ULID
|
||||
mock_request.mode = CorrectionMode.REVERT
|
||||
mock_request.target_decision_id = _DECISION_ULID
|
||||
mock_request.guidance = "Use FastAPI instead"
|
||||
mock_request.status = CorrectionStatus.PENDING
|
||||
|
||||
mock_impact = MagicMock()
|
||||
mock_impact.affected_decisions = [_DECISION_ULID]
|
||||
mock_impact.affected_files = ["src/main.py"]
|
||||
mock_impact.estimated_cost = "low"
|
||||
mock_impact.risk_level = "low"
|
||||
|
||||
mock_correction_svc.request_correction.return_value = mock_request
|
||||
mock_correction_svc.analyze_impact.return_value = mock_impact
|
||||
|
||||
self._patcher = patch(
|
||||
"cleveragents.application.services.correction_service.CorrectionService",
|
||||
return_value=mock_correction_svc,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_correct_dry_run(self) -> None:
|
||||
"""Benchmark plan correct dry-run command."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"revert",
|
||||
"--guidance",
|
||||
"Use FastAPI instead",
|
||||
"--dry-run",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class M3FixtureLoadSuite:
|
||||
"""Benchmark loading M3 fixture files."""
|
||||
|
||||
def time_load_decision_tree_fixture(self) -> None:
|
||||
"""Benchmark loading decision_tree_outputs.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "decision_tree_outputs.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_validation_attachments_fixture(self) -> None:
|
||||
"""Benchmark loading validation_attachments.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "validation_attachments.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_invariant_configs_fixture(self) -> None:
|
||||
"""Benchmark loading invariant_configs.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "invariant_configs.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
@@ -456,6 +456,59 @@ Performance baselines for persistence test operations:
|
||||
|
||||
Run with: `nox -s benchmark`
|
||||
|
||||
### ASV Benchmarks (`benchmarks/decision_persistence_bench.py`)
|
||||
|
||||
Performance baselines for decision persistence (serialization round-trip) operations:
|
||||
|
||||
- `DecisionDumpRoundTripSuite.time_root_dump` -- Root decision model_dump
|
||||
- `DecisionDumpRoundTripSuite.time_root_validate` -- Root decision model_validate
|
||||
- `DecisionDumpRoundTripSuite.time_root_roundtrip` -- Root decision full round-trip
|
||||
- `DecisionDumpRoundTripSuite.time_child_roundtrip` -- Child decision full round-trip
|
||||
- `DecisionDumpRoundTripSuite.time_full_roundtrip` -- Fully-populated decision round-trip
|
||||
- `DecisionJsonRoundTripSuite.time_root_dump_json` -- Root decision model_dump_json
|
||||
- `DecisionJsonRoundTripSuite.time_root_json_roundtrip` -- Root decision JSON round-trip
|
||||
- `DecisionJsonRoundTripSuite.time_full_json_roundtrip` -- Full decision JSON round-trip
|
||||
- `DecisionJsonRoundTripSuite.time_root_json_validate` -- Root decision model_validate from JSON
|
||||
- `DecisionJsonRoundTripSuite.time_full_json_validate` -- Full decision model_validate from JSON
|
||||
- `DecisionSnapshotPersistenceSuite.time_empty_snapshot_roundtrip` -- Empty snapshot round-trip
|
||||
- `DecisionSnapshotPersistenceSuite.time_small_snapshot_roundtrip` -- Small snapshot round-trip
|
||||
- `DecisionSnapshotPersistenceSuite.time_large_snapshot_roundtrip` -- Large snapshot (50 resources) round-trip
|
||||
- `DecisionTreeSerializationSuite.time_small_tree_roundtrip` -- 3-node tree round-trip
|
||||
- `DecisionTreeSerializationSuite.time_medium_tree_roundtrip` -- 7-node tree round-trip
|
||||
- `DecisionTreeSerializationSuite.time_large_tree_roundtrip` -- 15-node tree round-trip
|
||||
- `DecisionTreeSerializationSuite.time_large_tree_json_roundtrip` -- 15-node tree JSON round-trip
|
||||
|
||||
Run with: `nox -s benchmark`
|
||||
|
||||
### Behave: Decision Persistence (`features/decision_persistence.feature`)
|
||||
|
||||
21 scenarios covering Decision serialization round-trips, context-snapshot
|
||||
persistence, correction-chain reconstruction, and decision-tree
|
||||
reconstruction from serialized data:
|
||||
|
||||
- **CRUD round-trips**: Root and child decisions survive `model_dump` / `model_validate`.
|
||||
- **JSON round-trips**: Decisions with artifacts survive `model_dump_json` / `model_validate`.
|
||||
- **Snapshot persistence**: Empty snapshots, snapshots with resources, and actor state refs.
|
||||
- **Correction chains**: Correction metadata, superseded_by, and 3-decision chain reconstruction.
|
||||
- **Tree reconstruction**: 3-level tree (7 nodes) with parent-link integrity verification.
|
||||
- **Edge cases**: Empty alternatives, confidence boundaries (0.0, 1.0, None), all 11 types, long rationale.
|
||||
- **Downstream IDs**: `downstream_decision_ids` and `downstream_plan_ids` survive round-trip.
|
||||
|
||||
Step definitions: `features/steps/decision_persistence_steps.py`
|
||||
|
||||
### Robot Framework: Decision Persistence (`robot/decision_persistence.robot`)
|
||||
|
||||
7 integration smoke tests exercising decision persistence round-trips via
|
||||
a helper script (`robot/helper_decision_persistence.py`):
|
||||
|
||||
- Root decision round-trip
|
||||
- Child decision round-trip
|
||||
- Context snapshot persistence
|
||||
- JSON serialization round-trip
|
||||
- Correction chain persistence
|
||||
- Decision tree reconstruction
|
||||
- All 11 decision types round-trip
|
||||
|
||||
### Behave: Skill Registry Persistence (`features/skill_registry.feature`)
|
||||
|
||||
23 scenarios covering `SkillRepository` and `SkillRegistryService` operations:
|
||||
@@ -1090,3 +1143,61 @@ To perform a quick M1 source-code smoke run:
|
||||
- **Coverage drops**: The M1 smoke tests cover fixture loading, CLI argument
|
||||
parsing, and mock service integration. If coverage drops, look at
|
||||
`build/htmlcov/index.html` for uncovered lines in plan/action CLI commands.
|
||||
|
||||
## M3 Decision Tree, Validation Gating, and Invariant Enforcement Smoke Tests
|
||||
|
||||
### Overview
|
||||
|
||||
The M3 smoke suite validates the decision tree, correction workflow,
|
||||
validation gating (add / attach / detach), and invariant constraint
|
||||
(add / list / remove) subsystems via CLI integration tests.
|
||||
|
||||
Issue: [#179](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/179)
|
||||
|
||||
### Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|------|
|
||||
| `features/m3_decision_validation_smoke.feature` | Behave BDD scenarios (25 scenarios) |
|
||||
| `features/steps/m3_decision_validation_smoke_steps.py` | Step implementations |
|
||||
| `features/fixtures/m3/decision_tree_outputs.json` | Decision tree fixture data |
|
||||
| `features/fixtures/m3/validation_attachments.json` | Validation attachment fixture data |
|
||||
| `features/fixtures/m3/invariant_configs.json` | Invariant config fixture data |
|
||||
| `robot/m3_decision_validation_smoke.robot` | Robot Framework integration tests (8 cases) |
|
||||
| `robot/helper_m3_decision_validation_smoke.py` | Robot helper with 8 subcommands |
|
||||
| `benchmarks/m3_smoke_bench.py` | ASV benchmark suites (5 classes) |
|
||||
|
||||
### Scenario Categories
|
||||
|
||||
1. **Decision tree fixture loading** -- Loads JSON fixtures and verifies
|
||||
decision tree structure, invariant nodes, and correction metadata.
|
||||
2. **Validation fixture loading** -- Verifies validation attachment data
|
||||
including required and informational modes.
|
||||
3. **Invariant fixture loading** -- Verifies invariant config data
|
||||
including merge set precedence.
|
||||
4. **Invariant CLI** -- Tests `agents invariant add`, `list`, and `remove`
|
||||
commands with global and project scopes.
|
||||
5. **Validation CLI** -- Tests `agents validation add`, `attach`, and
|
||||
`detach` commands.
|
||||
6. **Plan correct CLI** -- Tests `agents plan correct` with `--dry-run`,
|
||||
`--mode revert`, and `--mode append`.
|
||||
7. **Negative cases** -- Empty guidance, invalid correction mode.
|
||||
|
||||
### Mock Strategy
|
||||
|
||||
- **Invariant CLI** patches `cleveragents.cli.commands.invariant._get_service`
|
||||
- **Validation CLI** patches `cleveragents.cli.commands.validation._get_tool_registry_service`
|
||||
- **Plan CLI** patches `cleveragents.cli.commands.plan._get_lifecycle_service`
|
||||
- **Correction** patches `CorrectionService` constructor at module level
|
||||
|
||||
### Failure Triage Tips
|
||||
|
||||
- **`AmbiguousStep` errors**: All M3 smoke steps are prefixed with `m3 smoke`.
|
||||
If ambiguous, check that no other step file defines a conflicting pattern.
|
||||
- **Fixture file not found**: Verify `features/fixtures/m3/` contains all
|
||||
three JSON fixture files.
|
||||
- **Robot `FAIL` sentinel**: Each Robot helper subcommand prints a detailed
|
||||
`FAIL:` line with exit code and output. Check the Robot log.
|
||||
- **Coverage drops**: The M3 smoke tests cover invariant/validation CLI
|
||||
commands, plan correct workflow, and fixture loading. Check
|
||||
`build/htmlcov/index.html` for uncovered lines.
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "minimal_decision_tree",
|
||||
"description": "A minimal decision tree with root prompt and one strategy choice",
|
||||
"decisions": [
|
||||
{
|
||||
"decision_type": "prompt_definition",
|
||||
"sequence_number": 0,
|
||||
"question": "What should we build?",
|
||||
"chosen_option": "A REST API for user management",
|
||||
"confidence_score": 0.95,
|
||||
"is_root": true
|
||||
},
|
||||
{
|
||||
"decision_type": "strategy_choice",
|
||||
"sequence_number": 1,
|
||||
"question": "Which framework to use?",
|
||||
"chosen_option": "FastAPI",
|
||||
"alternatives_considered": ["Flask", "Django"],
|
||||
"confidence_score": 0.85,
|
||||
"is_root": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tree_with_invariant",
|
||||
"description": "Decision tree with an invariant enforcement node",
|
||||
"decisions": [
|
||||
{
|
||||
"decision_type": "prompt_definition",
|
||||
"sequence_number": 0,
|
||||
"question": "Plan the database migration",
|
||||
"chosen_option": "Migrate user table to new schema",
|
||||
"confidence_score": 0.90,
|
||||
"is_root": true
|
||||
},
|
||||
{
|
||||
"decision_type": "invariant_enforced",
|
||||
"sequence_number": 1,
|
||||
"question": "Should production data be preserved?",
|
||||
"chosen_option": "Yes, all existing data must survive migration",
|
||||
"confidence_score": 1.0,
|
||||
"is_root": false,
|
||||
"invariant_text": "Never delete production data"
|
||||
},
|
||||
{
|
||||
"decision_type": "strategy_choice",
|
||||
"sequence_number": 2,
|
||||
"question": "Migration strategy?",
|
||||
"chosen_option": "Blue-green deployment with rollback",
|
||||
"alternatives_considered": ["In-place migration", "Shadow writes"],
|
||||
"confidence_score": 0.80,
|
||||
"is_root": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tree_with_correction",
|
||||
"description": "Decision tree that includes a correction decision superseding a prior choice",
|
||||
"decisions": [
|
||||
{
|
||||
"decision_type": "prompt_definition",
|
||||
"sequence_number": 0,
|
||||
"question": "Implement authentication",
|
||||
"chosen_option": "JWT-based auth with refresh tokens",
|
||||
"confidence_score": 0.90,
|
||||
"is_root": true
|
||||
},
|
||||
{
|
||||
"decision_type": "strategy_choice",
|
||||
"sequence_number": 1,
|
||||
"question": "Token storage approach?",
|
||||
"chosen_option": "HttpOnly cookies",
|
||||
"alternatives_considered": ["localStorage", "sessionStorage"],
|
||||
"confidence_score": 0.75,
|
||||
"is_root": false,
|
||||
"is_superseded": true
|
||||
},
|
||||
{
|
||||
"decision_type": "strategy_choice",
|
||||
"sequence_number": 2,
|
||||
"question": "Token storage approach? (corrected)",
|
||||
"chosen_option": "localStorage with XSS mitigation",
|
||||
"alternatives_considered": ["HttpOnly cookies", "sessionStorage"],
|
||||
"confidence_score": 0.82,
|
||||
"is_root": false,
|
||||
"is_correction": true,
|
||||
"correction_reason": "HttpOnly cookies cause issues with SSR"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "global_invariant",
|
||||
"description": "A system-wide invariant constraint",
|
||||
"invariant": {
|
||||
"text": "Never delete production data",
|
||||
"scope": "global",
|
||||
"source_name": "system"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "project_invariant",
|
||||
"description": "A project-scoped invariant constraint",
|
||||
"invariant": {
|
||||
"text": "All API changes need tests",
|
||||
"scope": "project",
|
||||
"source_name": "local/m3-smoke-proj"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "plan_invariant",
|
||||
"description": "A plan-scoped invariant",
|
||||
"invariant": {
|
||||
"text": "Use only approved libraries",
|
||||
"scope": "plan",
|
||||
"source_name": "01M3SM0KE00000000000000001"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "invariant_set_merge",
|
||||
"description": "Invariants for merge precedence testing",
|
||||
"global_invariants": [
|
||||
{
|
||||
"text": "Never delete production data",
|
||||
"scope": "global",
|
||||
"source_name": "system"
|
||||
},
|
||||
{
|
||||
"text": "All changes must be reversible",
|
||||
"scope": "global",
|
||||
"source_name": "system"
|
||||
}
|
||||
],
|
||||
"project_invariants": [
|
||||
{
|
||||
"text": "All API changes need tests",
|
||||
"scope": "project",
|
||||
"source_name": "local/m3-smoke-proj"
|
||||
},
|
||||
{
|
||||
"text": "Never delete production data",
|
||||
"scope": "project",
|
||||
"source_name": "local/m3-smoke-proj"
|
||||
}
|
||||
],
|
||||
"plan_invariants": [
|
||||
{
|
||||
"text": "Use only approved libraries",
|
||||
"scope": "plan",
|
||||
"source_name": "01M3SM0KE00000000000000001"
|
||||
}
|
||||
],
|
||||
"expected_merged_count": 4,
|
||||
"expected_dedup_text": "Never delete production data",
|
||||
"expected_dedup_scope": "plan"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "coverage_validation",
|
||||
"description": "A coverage-check validation with required mode",
|
||||
"validation": {
|
||||
"name": "local/coverage-check",
|
||||
"description": "Check code coverage meets threshold",
|
||||
"source": "custom",
|
||||
"mode": "required",
|
||||
"code": "def run(inputs): return {'passed': inputs['coverage'] >= 80}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lint_validation",
|
||||
"description": "A lint-check validation with informational mode",
|
||||
"validation": {
|
||||
"name": "local/lint-check",
|
||||
"description": "Check lint results",
|
||||
"source": "custom",
|
||||
"mode": "informational",
|
||||
"code": "def run(inputs): return {'passed': inputs['warnings'] == 0}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "attachment_required",
|
||||
"description": "A validation attachment in required mode",
|
||||
"attachment": {
|
||||
"validation_name": "local/coverage-check",
|
||||
"resource_id": "git-checkout/my-repo",
|
||||
"mode": "required",
|
||||
"project_name": "local/m3-smoke-proj"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "attachment_informational",
|
||||
"description": "A validation attachment in informational mode",
|
||||
"attachment": {
|
||||
"validation_name": "local/lint-check",
|
||||
"resource_id": "git-checkout/my-repo",
|
||||
"mode": "informational",
|
||||
"project_name": null,
|
||||
"plan_id": "01M3SM0KE00000000000000001"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
Feature: M3 decision tree, validation gating, and invariant enforcement smoke tests
|
||||
As a developer working with the CleverAgents M3 milestone
|
||||
I want to verify decision trees, corrections, validation lifecycle,
|
||||
and invariant constraints work end-to-end via the CLI
|
||||
So that the M3 foundation for decision + validation + invariant is solid
|
||||
|
||||
Background:
|
||||
Given a m3 smoke test runner
|
||||
And a m3 smoke mocked environment
|
||||
|
||||
# --- Decision tree fixture loading ---
|
||||
|
||||
Scenario: M3 smoke load decision tree fixture
|
||||
When I m3 smoke load the decision tree fixture
|
||||
Then the m3 smoke decision tree fixture should have a minimal tree entry
|
||||
And the m3 smoke minimal tree should contain 2 decisions
|
||||
|
||||
Scenario: M3 smoke load decision tree with invariant node
|
||||
When I m3 smoke load the decision tree fixture
|
||||
Then the m3 smoke decision tree fixture should have an invariant tree entry
|
||||
And the m3 smoke invariant tree should contain an invariant_enforced decision
|
||||
|
||||
Scenario: M3 smoke load decision tree with correction
|
||||
When I m3 smoke load the decision tree fixture
|
||||
Then the m3 smoke decision tree fixture should have a correction tree entry
|
||||
And the m3 smoke correction tree should contain a superseded decision
|
||||
And the m3 smoke correction tree should contain a correction decision
|
||||
|
||||
# --- Validation fixture loading ---
|
||||
|
||||
Scenario: M3 smoke load validation attachment fixtures
|
||||
When I m3 smoke load the validation attachment fixture
|
||||
Then the m3 smoke validation fixture should have a coverage entry
|
||||
And the m3 smoke coverage validation should have mode "required"
|
||||
|
||||
Scenario: M3 smoke load validation attachment with informational mode
|
||||
When I m3 smoke load the validation attachment fixture
|
||||
Then the m3 smoke validation fixture should have a lint entry
|
||||
And the m3 smoke lint validation should have mode "informational"
|
||||
|
||||
# --- Invariant fixture loading ---
|
||||
|
||||
Scenario: M3 smoke load invariant config fixtures
|
||||
When I m3 smoke load the invariant config fixture
|
||||
Then the m3 smoke invariant fixture should have a global entry
|
||||
And the m3 smoke global invariant text should be "Never delete production data"
|
||||
|
||||
Scenario: M3 smoke load invariant merge set
|
||||
When I m3 smoke load the invariant config fixture
|
||||
Then the m3 smoke invariant fixture should have a merge set entry
|
||||
And the m3 smoke merge set should define expected merged count 4
|
||||
|
||||
# --- Invariant CLI: add ---
|
||||
|
||||
Scenario: M3 smoke invariant add global
|
||||
When I m3 smoke invoke invariant add with text "Never delete production data" and global scope
|
||||
Then the m3 smoke invariant add should succeed
|
||||
And the m3 smoke invariant output should contain "Never delete production data"
|
||||
|
||||
Scenario: M3 smoke invariant add project scoped
|
||||
When I m3 smoke invoke invariant add with text "All API changes need tests" and project "local/m3-smoke-proj"
|
||||
Then the m3 smoke invariant add should succeed
|
||||
|
||||
# --- Invariant CLI: list ---
|
||||
|
||||
Scenario: M3 smoke invariant list shows entries
|
||||
Given m3 smoke invariants have been added
|
||||
When I m3 smoke invoke invariant list
|
||||
Then the m3 smoke invariant list should succeed
|
||||
And the m3 smoke invariant list output should contain "Never delete production data"
|
||||
|
||||
Scenario: M3 smoke invariant list with project filter
|
||||
Given m3 smoke invariants have been added
|
||||
When I m3 smoke invoke invariant list with project filter "local/m3-smoke-proj"
|
||||
Then the m3 smoke invariant list should succeed
|
||||
|
||||
# --- Invariant CLI: remove ---
|
||||
|
||||
Scenario: M3 smoke invariant remove by id
|
||||
Given m3 smoke invariants have been added
|
||||
When I m3 smoke invoke invariant remove with a known id
|
||||
Then the m3 smoke invariant remove should succeed
|
||||
|
||||
Scenario: M3 smoke invariant remove unknown id fails
|
||||
When I m3 smoke invoke invariant remove with id "01UNKNOWN000000000000000000"
|
||||
Then the m3 smoke invariant remove should fail
|
||||
|
||||
# --- Validation CLI: add ---
|
||||
|
||||
Scenario: M3 smoke validation add from config
|
||||
Given a m3 smoke temporary validation config file
|
||||
When I m3 smoke invoke validation add with the config
|
||||
Then the m3 smoke validation add should succeed
|
||||
And the m3 smoke validation output should contain "coverage-check"
|
||||
|
||||
# --- Validation CLI: attach ---
|
||||
|
||||
Scenario: M3 smoke validation attach to resource
|
||||
Given a m3 smoke validation "local/coverage-check" is registered
|
||||
When I m3 smoke invoke validation attach "local/coverage-check" to "git-checkout/my-repo"
|
||||
Then the m3 smoke validation attach should succeed
|
||||
|
||||
# --- Validation CLI: detach ---
|
||||
|
||||
Scenario: M3 smoke validation detach by id
|
||||
Given a m3 smoke validation attachment exists
|
||||
When I m3 smoke invoke validation detach with the attachment id
|
||||
Then the m3 smoke validation detach should succeed
|
||||
|
||||
# --- Plan correct CLI ---
|
||||
|
||||
Scenario: M3 smoke plan correct dry run
|
||||
Given a m3 smoke plan with decisions exists
|
||||
When I m3 smoke invoke plan correct in dry-run mode
|
||||
Then the m3 smoke plan correct dry run should succeed
|
||||
And the m3 smoke correction output should contain "Dry Run"
|
||||
|
||||
Scenario: M3 smoke plan correct revert executes
|
||||
Given a m3 smoke plan with decisions exists
|
||||
When I m3 smoke invoke plan correct in revert mode
|
||||
Then the m3 smoke plan correct should succeed
|
||||
|
||||
Scenario: M3 smoke plan correct append executes
|
||||
Given a m3 smoke plan with decisions exists
|
||||
When I m3 smoke invoke plan correct in append mode
|
||||
Then the m3 smoke plan correct should succeed
|
||||
|
||||
# --- Negative cases ---
|
||||
|
||||
Scenario: M3 smoke plan correct with empty guidance fails
|
||||
When I m3 smoke invoke plan correct with empty guidance
|
||||
Then the m3 smoke plan correct should fail
|
||||
|
||||
Scenario: M3 smoke plan correct with invalid mode fails
|
||||
When I m3 smoke invoke plan correct with invalid mode "rollback"
|
||||
Then the m3 smoke plan correct should fail
|
||||
@@ -0,0 +1,757 @@
|
||||
"""Step definitions for M3 decision tree, validation, and invariant smoke tests.
|
||||
|
||||
All step names are prefixed with ``m3 smoke`` to avoid ``AmbiguousStep``
|
||||
conflicts with existing steps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands.invariant import app as invariant_app
|
||||
from cleveragents.cli.commands.plan import app as plan_app
|
||||
from cleveragents.cli.commands.validation import app as validation_app
|
||||
from cleveragents.domain.models.core.correction import (
|
||||
CorrectionMode,
|
||||
CorrectionRequest,
|
||||
CorrectionStatus,
|
||||
)
|
||||
from cleveragents.domain.models.core.invariant import Invariant, InvariantScope
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m3"
|
||||
_PLAN_ULID = "01M3SM0KE00000000000000001"
|
||||
_DECISION_ULID = "01M3DEC1S10N0000000000001"
|
||||
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
|
||||
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
|
||||
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m3 smoke test runner")
|
||||
def step_m3_smoke_runner(context: Context) -> None:
|
||||
"""Set up the CLI runner for M3 smoke tests."""
|
||||
context.runner = CliRunner()
|
||||
|
||||
|
||||
@given("a m3 smoke mocked environment")
|
||||
def step_m3_smoke_mock_env(context: Context) -> None:
|
||||
"""Set up the mocked services for M3 smoke tests."""
|
||||
context.mock_invariant_service = MagicMock()
|
||||
context.mock_tool_registry_service = MagicMock()
|
||||
context.mock_correction_service = MagicMock()
|
||||
context.mock_lifecycle_service = MagicMock()
|
||||
|
||||
context.invariant_patcher = patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=context.mock_invariant_service,
|
||||
)
|
||||
context.validation_patcher = patch(
|
||||
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
||||
return_value=context.mock_tool_registry_service,
|
||||
)
|
||||
context.plan_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=context.mock_lifecycle_service,
|
||||
)
|
||||
|
||||
context.invariant_patcher.start()
|
||||
context.validation_patcher.start()
|
||||
context.plan_patcher.start()
|
||||
|
||||
context.last_result = None
|
||||
context._m3_known_invariant_id = None
|
||||
context._m3_known_attachment_id = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decision tree fixture loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m3 smoke load the decision tree fixture")
|
||||
def step_m3_load_decision_tree(context: Context) -> None:
|
||||
"""Load the decision tree outputs fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "decision_tree_outputs.json"
|
||||
with open(fixture_path) as f:
|
||||
context.decision_tree_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m3 smoke decision tree fixture should have a minimal tree entry")
|
||||
def step_m3_has_minimal_tree(context: Context) -> None:
|
||||
"""Verify fixture has a minimal_decision_tree entry."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "minimal_decision_tree" in names, (
|
||||
f"Expected 'minimal_decision_tree' in {names}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m3 smoke minimal tree should contain {count:d} decisions")
|
||||
def step_m3_minimal_tree_count(context: Context, count: int) -> None:
|
||||
"""Verify the minimal tree has the expected number of decisions."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
minimal = next(f for f in fixtures if f["name"] == "minimal_decision_tree")
|
||||
actual = len(minimal["decisions"])
|
||||
assert actual == count, f"Expected {count} decisions, got {actual}"
|
||||
|
||||
|
||||
@then("the m3 smoke decision tree fixture should have an invariant tree entry")
|
||||
def step_m3_has_invariant_tree(context: Context) -> None:
|
||||
"""Verify fixture has a tree_with_invariant entry."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "tree_with_invariant" in names, f"Expected 'tree_with_invariant' in {names}"
|
||||
|
||||
|
||||
@then("the m3 smoke invariant tree should contain an invariant_enforced decision")
|
||||
def step_m3_invariant_tree_has_enforced(context: Context) -> None:
|
||||
"""Verify the invariant tree has an invariant_enforced decision."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
tree = next(f for f in fixtures if f["name"] == "tree_with_invariant")
|
||||
types = [d["decision_type"] for d in tree["decisions"]]
|
||||
assert "invariant_enforced" in types, f"Expected 'invariant_enforced' in {types}"
|
||||
|
||||
|
||||
@then("the m3 smoke decision tree fixture should have a correction tree entry")
|
||||
def step_m3_has_correction_tree(context: Context) -> None:
|
||||
"""Verify fixture has a tree_with_correction entry."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "tree_with_correction" in names, (
|
||||
f"Expected 'tree_with_correction' in {names}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m3 smoke correction tree should contain a superseded decision")
|
||||
def step_m3_correction_tree_has_superseded(context: Context) -> None:
|
||||
"""Verify the correction tree has a superseded decision."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
tree = next(f for f in fixtures if f["name"] == "tree_with_correction")
|
||||
superseded = [d for d in tree["decisions"] if d.get("is_superseded")]
|
||||
assert len(superseded) > 0, "Expected at least one superseded decision"
|
||||
|
||||
|
||||
@then("the m3 smoke correction tree should contain a correction decision")
|
||||
def step_m3_correction_tree_has_correction(context: Context) -> None:
|
||||
"""Verify the correction tree has a correction decision."""
|
||||
fixtures = context.decision_tree_fixtures["fixtures"]
|
||||
tree = next(f for f in fixtures if f["name"] == "tree_with_correction")
|
||||
corrections = [d for d in tree["decisions"] if d.get("is_correction")]
|
||||
assert len(corrections) > 0, "Expected at least one correction decision"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation fixture loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m3 smoke load the validation attachment fixture")
|
||||
def step_m3_load_validation_fixtures(context: Context) -> None:
|
||||
"""Load the validation attachments fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "validation_attachments.json"
|
||||
with open(fixture_path) as f:
|
||||
context.validation_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m3 smoke validation fixture should have a coverage entry")
|
||||
def step_m3_has_coverage_validation(context: Context) -> None:
|
||||
"""Verify fixture has a coverage_validation entry."""
|
||||
fixtures = context.validation_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "coverage_validation" in names, f"Expected 'coverage_validation' in {names}"
|
||||
|
||||
|
||||
@then('the m3 smoke coverage validation should have mode "{mode}"')
|
||||
def step_m3_coverage_mode(context: Context, mode: str) -> None:
|
||||
"""Verify coverage validation has expected mode."""
|
||||
fixtures = context.validation_fixtures["fixtures"]
|
||||
entry = next(f for f in fixtures if f["name"] == "coverage_validation")
|
||||
actual = entry["validation"]["mode"]
|
||||
assert actual == mode, f"Expected mode '{mode}', got '{actual}'"
|
||||
|
||||
|
||||
@then("the m3 smoke validation fixture should have a lint entry")
|
||||
def step_m3_has_lint_validation(context: Context) -> None:
|
||||
"""Verify fixture has a lint_validation entry."""
|
||||
fixtures = context.validation_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "lint_validation" in names, f"Expected 'lint_validation' in {names}"
|
||||
|
||||
|
||||
@then('the m3 smoke lint validation should have mode "{mode}"')
|
||||
def step_m3_lint_mode(context: Context, mode: str) -> None:
|
||||
"""Verify lint validation has expected mode."""
|
||||
fixtures = context.validation_fixtures["fixtures"]
|
||||
entry = next(f for f in fixtures if f["name"] == "lint_validation")
|
||||
actual = entry["validation"]["mode"]
|
||||
assert actual == mode, f"Expected mode '{mode}', got '{actual}'"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invariant fixture loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m3 smoke load the invariant config fixture")
|
||||
def step_m3_load_invariant_fixtures(context: Context) -> None:
|
||||
"""Load the invariant configs fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "invariant_configs.json"
|
||||
with open(fixture_path) as f:
|
||||
context.invariant_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m3 smoke invariant fixture should have a global entry")
|
||||
def step_m3_has_global_invariant(context: Context) -> None:
|
||||
"""Verify fixture has a global_invariant entry."""
|
||||
fixtures = context.invariant_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "global_invariant" in names, f"Expected 'global_invariant' in {names}"
|
||||
|
||||
|
||||
@then('the m3 smoke global invariant text should be "{text}"')
|
||||
def step_m3_global_invariant_text(context: Context, text: str) -> None:
|
||||
"""Verify global invariant has expected text."""
|
||||
fixtures = context.invariant_fixtures["fixtures"]
|
||||
entry = next(f for f in fixtures if f["name"] == "global_invariant")
|
||||
actual = entry["invariant"]["text"]
|
||||
assert actual == text, f"Expected text '{text}', got '{actual}'"
|
||||
|
||||
|
||||
@then("the m3 smoke invariant fixture should have a merge set entry")
|
||||
def step_m3_has_merge_set(context: Context) -> None:
|
||||
"""Verify fixture has an invariant_set_merge entry."""
|
||||
fixtures = context.invariant_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "invariant_set_merge" in names, f"Expected 'invariant_set_merge' in {names}"
|
||||
|
||||
|
||||
@then("the m3 smoke merge set should define expected merged count {count:d}")
|
||||
def step_m3_merge_set_count(context: Context, count: int) -> None:
|
||||
"""Verify merge set defines expected merged count."""
|
||||
fixtures = context.invariant_fixtures["fixtures"]
|
||||
entry = next(f for f in fixtures if f["name"] == "invariant_set_merge")
|
||||
actual = entry["expected_merged_count"]
|
||||
assert actual == count, f"Expected merged count {count}, got {actual}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invariant CLI: add
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_m3_invariant(
|
||||
*,
|
||||
text: str = "Never delete production data",
|
||||
scope: InvariantScope = InvariantScope.GLOBAL,
|
||||
source_name: str = "system",
|
||||
) -> Invariant:
|
||||
"""Create an Invariant instance for M3 smoke tests."""
|
||||
return Invariant(
|
||||
id=_INVARIANT_ULID,
|
||||
text=text,
|
||||
scope=scope,
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
|
||||
@when('I m3 smoke invoke invariant add with text "{text}" and global scope')
|
||||
def step_m3_invariant_add_global(context: Context, text: str) -> None:
|
||||
"""Invoke invariant add with global scope."""
|
||||
context.mock_invariant_service.add_invariant.return_value = _make_m3_invariant(
|
||||
text=text
|
||||
)
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["add", "--global", text, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m3 smoke invoke invariant add with text "{text}" and project "{project}"')
|
||||
def step_m3_invariant_add_project(context: Context, text: str, project: str) -> None:
|
||||
"""Invoke invariant add with project scope."""
|
||||
context.mock_invariant_service.add_invariant.return_value = _make_m3_invariant(
|
||||
text=text,
|
||||
scope=InvariantScope.PROJECT,
|
||||
source_name=project,
|
||||
)
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["add", "--project", project, text, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke invariant add should succeed")
|
||||
def step_m3_invariant_add_ok(context: Context) -> None:
|
||||
"""Verify invariant add succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m3 smoke invariant output should contain "{text}"')
|
||||
def step_m3_invariant_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify invariant output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invariant CLI: list
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("m3 smoke invariants have been added")
|
||||
def step_m3_invariants_added(context: Context) -> None:
|
||||
"""Set up mock invariant service with pre-added invariants."""
|
||||
inv1 = _make_m3_invariant()
|
||||
inv2 = _make_m3_invariant(
|
||||
text="All API changes need tests",
|
||||
scope=InvariantScope.PROJECT,
|
||||
source_name="local/m3-smoke-proj",
|
||||
)
|
||||
context.mock_invariant_service.list_invariants.return_value = [inv1, inv2]
|
||||
context._m3_known_invariant_id = _INVARIANT_ULID
|
||||
|
||||
|
||||
@when("I m3 smoke invoke invariant list")
|
||||
def step_m3_invariant_list(context: Context) -> None:
|
||||
"""Invoke invariant list."""
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["list", "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m3 smoke invoke invariant list with project filter "{project}"')
|
||||
def step_m3_invariant_list_project(context: Context, project: str) -> None:
|
||||
"""Invoke invariant list with project filter."""
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["list", "--project", project, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke invariant list should succeed")
|
||||
def step_m3_invariant_list_ok(context: Context) -> None:
|
||||
"""Verify invariant list succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m3 smoke invariant list output should contain "{text}"')
|
||||
def step_m3_invariant_list_contains(context: Context, text: str) -> None:
|
||||
"""Verify invariant list output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invariant CLI: remove
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m3 smoke invoke invariant remove with a known id")
|
||||
def step_m3_invariant_remove_known(context: Context) -> None:
|
||||
"""Invoke invariant remove with a known invariant ID."""
|
||||
inv_id = context._m3_known_invariant_id or _INVARIANT_ULID
|
||||
context.mock_invariant_service.remove_invariant.return_value = _make_m3_invariant()
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["remove", "--yes", inv_id, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m3 smoke invoke invariant remove with id "{inv_id}"')
|
||||
def step_m3_invariant_remove_by_id(context: Context, inv_id: str) -> None:
|
||||
"""Invoke invariant remove with specific id."""
|
||||
from cleveragents.core.exceptions import ResourceNotFoundError
|
||||
|
||||
context.mock_invariant_service.remove_invariant.side_effect = ResourceNotFoundError(
|
||||
message=f"Invariant not found: {inv_id}",
|
||||
resource_type="invariant",
|
||||
resource_id=inv_id,
|
||||
)
|
||||
result = context.runner.invoke(
|
||||
invariant_app,
|
||||
["remove", "--yes", inv_id, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke invariant remove should succeed")
|
||||
def step_m3_invariant_remove_ok(context: Context) -> None:
|
||||
"""Verify invariant remove succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m3 smoke invariant remove should fail")
|
||||
def step_m3_invariant_remove_fail(context: Context) -> None:
|
||||
"""Verify invariant remove failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation CLI: add
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m3 smoke temporary validation config file")
|
||||
def step_m3_temp_validation_config(context: Context) -> None:
|
||||
"""Create a temp YAML config file for validation add."""
|
||||
config_content = (
|
||||
"name: local/coverage-check\n"
|
||||
"description: Check code coverage meets threshold\n"
|
||||
"source: custom\n"
|
||||
"mode: required\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': inputs['coverage'] >= 80}\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
context.m3_validation_config = tmp.name
|
||||
mock_validation = MagicMock()
|
||||
mock_validation.name = "local/coverage-check"
|
||||
mock_validation.as_cli_dict.return_value = {
|
||||
"name": "local/coverage-check",
|
||||
"description": "Check code coverage meets threshold",
|
||||
"source": "custom",
|
||||
"mode": "required",
|
||||
}
|
||||
context.mock_tool_registry_service.register_tool.return_value = mock_validation
|
||||
|
||||
|
||||
@when("I m3 smoke invoke validation add with the config")
|
||||
def step_m3_validation_add(context: Context) -> None:
|
||||
"""Invoke validation add CLI with config file."""
|
||||
result = context.runner.invoke(
|
||||
validation_app,
|
||||
["add", "--config", context.m3_validation_config, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke validation add should succeed")
|
||||
def step_m3_validation_add_ok(context: Context) -> None:
|
||||
"""Verify validation add succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m3 smoke validation output should contain "{text}"')
|
||||
def step_m3_validation_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify validation output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation CLI: attach
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a m3 smoke validation "{name}" is registered')
|
||||
def step_m3_validation_registered(context: Context, name: str) -> None:
|
||||
"""Set up a registered validation for attach tests."""
|
||||
mock_attachment = MagicMock()
|
||||
mock_attachment.attachment_id = _ATTACHMENT_ULID
|
||||
mock_attachment.validation_name = name
|
||||
mock_attachment.resource_id = "git-checkout/my-repo"
|
||||
mock_attachment.mode = "required"
|
||||
mock_attachment.project_name = None
|
||||
mock_attachment.plan_id = None
|
||||
mock_attachment.created_at = "2026-01-01T00:00:00"
|
||||
context.mock_tool_registry_service.attach_validation.return_value = mock_attachment
|
||||
context._m3_known_attachment_id = _ATTACHMENT_ULID
|
||||
|
||||
|
||||
@when('I m3 smoke invoke validation attach "{name}" to "{resource}"')
|
||||
def step_m3_validation_attach(context: Context, name: str, resource: str) -> None:
|
||||
"""Invoke validation attach CLI."""
|
||||
result = context.runner.invoke(
|
||||
validation_app,
|
||||
["attach", resource, name, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke validation attach should succeed")
|
||||
def step_m3_validation_attach_ok(context: Context) -> None:
|
||||
"""Verify validation attach succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation CLI: detach
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m3 smoke validation attachment exists")
|
||||
def step_m3_attachment_exists(context: Context) -> None:
|
||||
"""Set up an existing validation attachment for detach tests."""
|
||||
context.mock_tool_registry_service.detach_validation.return_value = True
|
||||
context._m3_known_attachment_id = _ATTACHMENT_ULID
|
||||
|
||||
|
||||
@when("I m3 smoke invoke validation detach with the attachment id")
|
||||
def step_m3_validation_detach(context: Context) -> None:
|
||||
"""Invoke validation detach CLI."""
|
||||
att_id = context._m3_known_attachment_id or _ATTACHMENT_ULID
|
||||
result = context.runner.invoke(
|
||||
validation_app,
|
||||
["detach", "--yes", att_id, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke validation detach should succeed")
|
||||
def step_m3_validation_detach_ok(context: Context) -> None:
|
||||
"""Verify validation detach succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan correct CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m3 smoke plan with decisions exists")
|
||||
def step_m3_plan_with_decisions(context: Context) -> None:
|
||||
"""Set up mock correction service for plan correct tests."""
|
||||
mock_request = MagicMock(spec=CorrectionRequest)
|
||||
mock_request.correction_id = _CORRECTION_ULID
|
||||
mock_request.mode = CorrectionMode.REVERT
|
||||
mock_request.target_decision_id = _DECISION_ULID
|
||||
mock_request.guidance = "Use FastAPI instead"
|
||||
mock_request.status = CorrectionStatus.PENDING
|
||||
|
||||
mock_impact = MagicMock()
|
||||
mock_impact.affected_decisions = [_DECISION_ULID]
|
||||
mock_impact.affected_files = ["src/main.py"]
|
||||
mock_impact.estimated_cost = "low"
|
||||
mock_impact.risk_level = "low"
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.correction_id = _CORRECTION_ULID
|
||||
mock_result.status = CorrectionStatus.APPLIED
|
||||
mock_result.new_decisions = ["01M3NEWDEC000000000000001"]
|
||||
mock_result.reverted_decisions = [_DECISION_ULID]
|
||||
|
||||
context.mock_correction_service.request_correction.return_value = mock_request
|
||||
context.mock_correction_service.analyze_impact.return_value = mock_impact
|
||||
context.mock_correction_service.execute_correction.return_value = mock_result
|
||||
|
||||
context.correction_patcher = patch(
|
||||
"cleveragents.application.services.correction_service.CorrectionService",
|
||||
return_value=context.mock_correction_service,
|
||||
)
|
||||
context.correction_patcher.start()
|
||||
|
||||
|
||||
@when("I m3 smoke invoke plan correct in dry-run mode")
|
||||
def step_m3_plan_correct_dry_run(context: Context) -> None:
|
||||
"""Invoke plan correct with --dry-run."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"revert",
|
||||
"--guidance",
|
||||
"Use FastAPI instead",
|
||||
"--dry-run",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when("I m3 smoke invoke plan correct in revert mode")
|
||||
def step_m3_plan_correct_revert(context: Context) -> None:
|
||||
"""Invoke plan correct with revert mode."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"revert",
|
||||
"--guidance",
|
||||
"Use FastAPI instead",
|
||||
"--yes",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when("I m3 smoke invoke plan correct in append mode")
|
||||
def step_m3_plan_correct_append(context: Context) -> None:
|
||||
"""Invoke plan correct with append mode."""
|
||||
context.mock_correction_service.request_correction.return_value.mode = (
|
||||
CorrectionMode.APPEND
|
||||
)
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"append",
|
||||
"--guidance",
|
||||
"Add caching layer",
|
||||
"--yes",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke plan correct dry run should succeed")
|
||||
def step_m3_correct_dry_run_ok(context: Context) -> None:
|
||||
"""Verify plan correct dry run succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m3 smoke correction output should contain "{text}"')
|
||||
def step_m3_correction_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify correction output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
@then("the m3 smoke plan correct should succeed")
|
||||
def step_m3_correct_ok(context: Context) -> None:
|
||||
"""Verify plan correct succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Negative cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m3 smoke invoke plan correct with empty guidance")
|
||||
def step_m3_correct_empty_guidance(context: Context) -> None:
|
||||
"""Invoke plan correct with empty guidance string."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"revert",
|
||||
"--guidance",
|
||||
"",
|
||||
"--yes",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m3 smoke invoke plan correct with invalid mode "{mode}"')
|
||||
def step_m3_correct_invalid_mode(context: Context, mode: str) -> None:
|
||||
"""Invoke plan correct with an invalid mode."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
mode,
|
||||
"--guidance",
|
||||
"Some guidance",
|
||||
"--yes",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m3 smoke plan correct should fail")
|
||||
def step_m3_correct_fail(context: Context) -> None:
|
||||
"""Verify plan correct failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def after_scenario(context: Context, scenario: object) -> None:
|
||||
"""Clean up patchers after each scenario."""
|
||||
for name in (
|
||||
"invariant_patcher",
|
||||
"validation_patcher",
|
||||
"plan_patcher",
|
||||
"correction_patcher",
|
||||
):
|
||||
patcher = getattr(context, name, None)
|
||||
if patcher:
|
||||
with contextlib.suppress(RuntimeError):
|
||||
patcher.stop()
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Helper script for m3_decision_validation_smoke.robot E2E tests.
|
||||
|
||||
Each subcommand is a self-contained check that prints a sentinel on success.
|
||||
Uses ``--format plain`` where possible to stabilise assertion strings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.invariant import app as invariant_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
|
||||
from cleveragents.domain.models.core.correction import ( # noqa: E402
|
||||
CorrectionMode,
|
||||
CorrectionStatus,
|
||||
)
|
||||
from cleveragents.domain.models.core.invariant import ( # noqa: E402
|
||||
Invariant,
|
||||
InvariantScope,
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
_PLAN_ULID = "01M3SM0KE00000000000000001"
|
||||
_DECISION_ULID = "01M3DEC1S10N0000000000001"
|
||||
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
|
||||
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
|
||||
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
|
||||
|
||||
|
||||
def _mock_invariant(
|
||||
*,
|
||||
text: str = "Never delete production data",
|
||||
scope: InvariantScope = InvariantScope.GLOBAL,
|
||||
source_name: str = "system",
|
||||
) -> Invariant:
|
||||
"""Create an Invariant for M3 smoke tests."""
|
||||
return Invariant(
|
||||
id=_INVARIANT_ULID,
|
||||
text=text,
|
||||
scope=scope,
|
||||
source_name=source_name,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def invariant_add_global() -> None:
|
||||
"""Add a global invariant constraint."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.add_invariant.return_value = _mock_invariant()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
invariant_app,
|
||||
["add", "--global", "Never delete production data", "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-invariant-add-global-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def invariant_add_project() -> None:
|
||||
"""Add a project-scoped invariant constraint."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.add_invariant.return_value = _mock_invariant(
|
||||
text="All API changes need tests",
|
||||
scope=InvariantScope.PROJECT,
|
||||
source_name="local/m3-smoke-proj",
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
invariant_app,
|
||||
[
|
||||
"add",
|
||||
"--project",
|
||||
"local/m3-smoke-proj",
|
||||
"All API changes need tests",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-invariant-add-project-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def invariant_list() -> None:
|
||||
"""List invariants."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.list_invariants.return_value = [
|
||||
_mock_invariant(),
|
||||
_mock_invariant(
|
||||
text="All API changes need tests",
|
||||
scope=InvariantScope.PROJECT,
|
||||
source_name="local/m3-smoke-proj",
|
||||
),
|
||||
]
|
||||
with patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
invariant_app,
|
||||
["list", "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-invariant-list-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def invariant_remove() -> None:
|
||||
"""Remove an invariant by ID."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.remove_invariant.return_value = _mock_invariant()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.invariant._get_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
invariant_app,
|
||||
["remove", "--yes", _INVARIANT_ULID, "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-invariant-remove-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validation_add() -> None:
|
||||
"""Register a validation from YAML config."""
|
||||
config_content = (
|
||||
"name: local/coverage-check\n"
|
||||
"description: Check code coverage meets threshold\n"
|
||||
"source: custom\n"
|
||||
"mode: required\n"
|
||||
"code: |\n"
|
||||
" def run(inputs):\n"
|
||||
" return {'passed': inputs['coverage'] >= 80}\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
tmp.write(config_content)
|
||||
tmp_name = tmp.name
|
||||
|
||||
mock_svc = MagicMock()
|
||||
mock_validation = MagicMock()
|
||||
mock_validation.name = "local/coverage-check"
|
||||
mock_validation.as_cli_dict.return_value = {
|
||||
"name": "local/coverage-check",
|
||||
"description": "Check code coverage meets threshold",
|
||||
"source": "custom",
|
||||
"mode": "required",
|
||||
}
|
||||
mock_svc.register_tool.return_value = mock_validation
|
||||
with patch(
|
||||
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
["add", "--config", tmp_name, "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0 and "coverage-check" in result.output:
|
||||
print("m3-validation-add-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validation_attach() -> None:
|
||||
"""Attach a validation to a resource."""
|
||||
mock_svc = MagicMock()
|
||||
mock_attachment = MagicMock()
|
||||
mock_attachment.attachment_id = _ATTACHMENT_ULID
|
||||
mock_attachment.validation_name = "local/coverage-check"
|
||||
mock_attachment.resource_id = "git-checkout/my-repo"
|
||||
mock_attachment.mode = "required"
|
||||
mock_attachment.project_name = None
|
||||
mock_attachment.plan_id = None
|
||||
mock_attachment.created_at = "2026-01-01T00:00:00"
|
||||
mock_svc.attach_validation.return_value = mock_attachment
|
||||
with patch(
|
||||
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
[
|
||||
"attach",
|
||||
"git-checkout/my-repo",
|
||||
"local/coverage-check",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-validation-attach-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validation_detach() -> None:
|
||||
"""Detach a validation from a resource."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.detach_validation.return_value = True
|
||||
with patch(
|
||||
"cleveragents.cli.commands.validation._get_tool_registry_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
validation_app,
|
||||
["detach", "--yes", _ATTACHMENT_ULID, "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-validation-detach-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_correct_dry_run() -> None:
|
||||
"""Run plan correct in dry-run mode."""
|
||||
mock_correction_svc = MagicMock()
|
||||
mock_request = MagicMock()
|
||||
mock_request.correction_id = _CORRECTION_ULID
|
||||
mock_request.mode = CorrectionMode.REVERT
|
||||
mock_request.target_decision_id = _DECISION_ULID
|
||||
mock_request.guidance = "Use FastAPI instead"
|
||||
mock_request.status = CorrectionStatus.PENDING
|
||||
|
||||
mock_impact = MagicMock()
|
||||
mock_impact.affected_decisions = [_DECISION_ULID]
|
||||
mock_impact.affected_files = ["src/main.py"]
|
||||
mock_impact.estimated_cost = "low"
|
||||
mock_impact.risk_level = "low"
|
||||
|
||||
mock_correction_svc.request_correction.return_value = mock_request
|
||||
mock_correction_svc.analyze_impact.return_value = mock_impact
|
||||
|
||||
with patch(
|
||||
"cleveragents.application.services.correction_service.CorrectionService",
|
||||
return_value=mock_correction_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"correct",
|
||||
_DECISION_ULID,
|
||||
"--mode",
|
||||
"revert",
|
||||
"--guidance",
|
||||
"Use FastAPI instead",
|
||||
"--dry-run",
|
||||
"--plan",
|
||||
_PLAN_ULID,
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m3-plan-correct-dry-run-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"invariant-add-global": invariant_add_global,
|
||||
"invariant-add-project": invariant_add_project,
|
||||
"invariant-list": invariant_list,
|
||||
"invariant-remove": invariant_remove,
|
||||
"validation-add": validation_add,
|
||||
"validation-attach": validation_attach,
|
||||
"validation-detach": validation_detach,
|
||||
"plan-correct-dry-run": plan_correct_dry_run,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
|
||||
sys.exit(1)
|
||||
fn = _COMMANDS[sys.argv[1]]
|
||||
fn() # type: ignore[operator]
|
||||
@@ -0,0 +1,73 @@
|
||||
*** Settings ***
|
||||
Documentation M3 decision tree, validation gating, and invariant enforcement E2E smoke tests via CLI
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m3_decision_validation_smoke.py
|
||||
|
||||
*** Test Cases ***
|
||||
M3 Invariant Add Global
|
||||
[Documentation] Add a global invariant constraint via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-add-global cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-invariant-add-global-ok
|
||||
|
||||
M3 Invariant Add Project Scoped
|
||||
[Documentation] Add a project-scoped invariant constraint via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-add-project cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-invariant-add-project-ok
|
||||
|
||||
M3 Invariant List
|
||||
[Documentation] List invariants via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-list cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-invariant-list-ok
|
||||
|
||||
M3 Invariant Remove
|
||||
[Documentation] Remove an invariant by ID via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-remove cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-invariant-remove-ok
|
||||
|
||||
M3 Validation Add From Config
|
||||
[Documentation] Register a validation from YAML config via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validation-add cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-validation-add-ok
|
||||
|
||||
M3 Validation Attach
|
||||
[Documentation] Attach a validation to a resource via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validation-attach cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-validation-attach-ok
|
||||
|
||||
M3 Validation Detach
|
||||
[Documentation] Detach a validation from a resource via CLI
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validation-detach cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-validation-detach-ok
|
||||
|
||||
M3 Plan Correct Dry Run
|
||||
[Documentation] Run plan correct in dry-run mode to preview correction impact
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-correct-dry-run cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m3-plan-correct-dry-run-ok
|
||||
Reference in New Issue
Block a user