test(e2e): add M3 decision + validation suites
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / quality (pull_request) Successful in 19s
CI / build (pull_request) Successful in 20s
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m5s
CI / integration_tests (pull_request) Successful in 4m4s
CI / benchmark-regression (pull_request) Successful in 22m21s
CI / unit_tests (pull_request) Failing after 34m7s
CI / docker (pull_request) Has been skipped
CI / coverage (pull_request) Has been cancelled

This commit is contained in:
2026-02-25 21:26:19 +00:00
parent b437fb8fd4
commit ace7311de4
9 changed files with 1860 additions and 0 deletions
+255
View File
@@ -0,0 +1,255 @@
"""ASV benchmarks for M3 decision tree, validation, and invariant CLI overhead.
Measures the performance of:
- Invariant add / list / remove CLI commands
- Validation add / attach / detach CLI commands
- Plan correct (dry-run) CLI command
- Decision tree fixture loading
- Invariant merge precedence computation
"""
from __future__ import annotations
import importlib
import json
import sys
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.invariant import app as invariant_app # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
from cleveragents.domain.models.core.correction import ( # noqa: E402
CorrectionMode,
CorrectionStatus,
)
from cleveragents.domain.models.core.invariant import ( # noqa: E402
Invariant,
InvariantScope,
)
_runner = CliRunner()
_PLAN_ULID = "01M3SM0KE00000000000000001"
_DECISION_ULID = "01M3DEC1S10N0000000000001"
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m3"
def _mock_invariant(
*,
text: str = "Never delete production data",
scope: InvariantScope = InvariantScope.GLOBAL,
source_name: str = "system",
) -> Invariant:
return Invariant(
id=_INVARIANT_ULID,
text=text,
scope=scope,
source_name=source_name,
)
class M3InvariantAddSuite:
"""Benchmark invariant add CLI command."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.add_invariant.return_value = _mock_invariant()
self._patcher = patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=self._mock_service,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_invariant_add_global(self) -> None:
"""Benchmark invariant add with global scope."""
_runner.invoke(
invariant_app,
["add", "--global", "Never delete production data", "--format", "plain"],
)
def time_invariant_add_project(self) -> None:
"""Benchmark invariant add with project scope."""
_runner.invoke(
invariant_app,
[
"add",
"--project",
"local/m3-smoke-proj",
"All API changes need tests",
"--format",
"plain",
],
)
class M3InvariantListSuite:
"""Benchmark invariant list CLI command."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.list_invariants.return_value = [
_mock_invariant(),
_mock_invariant(
text="All API changes need tests",
scope=InvariantScope.PROJECT,
source_name="local/m3-smoke-proj",
),
]
self._patcher = patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=self._mock_service,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_invariant_list(self) -> None:
"""Benchmark invariant list command."""
_runner.invoke(invariant_app, ["list", "--format", "plain"])
def time_invariant_list_with_project(self) -> None:
"""Benchmark invariant list with project filter."""
_runner.invoke(
invariant_app,
["list", "--project", "local/m3-smoke-proj", "--format", "plain"],
)
class M3ValidationAddSuite:
"""Benchmark validation add CLI command."""
def setup(self) -> None:
self._mock_service = MagicMock()
mock_validation = MagicMock()
mock_validation.name = "local/coverage-check"
mock_validation.as_cli_dict.return_value = {
"name": "local/coverage-check",
"description": "Check code coverage meets threshold",
"source": "custom",
"mode": "required",
}
self._mock_service.register_tool.return_value = mock_validation
self._patcher = patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=self._mock_service,
)
self._patcher.start()
config_content = (
"name: local/coverage-check\n"
"description: Check code coverage meets threshold\n"
"source: custom\n"
"mode: required\n"
"code: |\n"
" def run(inputs):\n"
" return {'passed': inputs['coverage'] >= 80}\n"
)
with tempfile.NamedTemporaryFile(
mode="w",
suffix=".yaml",
delete=False,
) as tmp:
tmp.write(config_content)
self._config_path = tmp.name
def teardown(self) -> None:
self._patcher.stop()
def time_validation_add(self) -> None:
"""Benchmark validation add from YAML config."""
_runner.invoke(
validation_app,
["add", "--config", self._config_path, "--format", "plain"],
)
class M3PlanCorrectSuite:
"""Benchmark plan correct (dry-run) CLI command."""
def setup(self) -> None:
mock_correction_svc = MagicMock()
mock_request = MagicMock()
mock_request.correction_id = _CORRECTION_ULID
mock_request.mode = CorrectionMode.REVERT
mock_request.target_decision_id = _DECISION_ULID
mock_request.guidance = "Use FastAPI instead"
mock_request.status = CorrectionStatus.PENDING
mock_impact = MagicMock()
mock_impact.affected_decisions = [_DECISION_ULID]
mock_impact.affected_files = ["src/main.py"]
mock_impact.estimated_cost = "low"
mock_impact.risk_level = "low"
mock_correction_svc.request_correction.return_value = mock_request
mock_correction_svc.analyze_impact.return_value = mock_impact
self._patcher = patch(
"cleveragents.application.services.correction_service.CorrectionService",
return_value=mock_correction_svc,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_plan_correct_dry_run(self) -> None:
"""Benchmark plan correct dry-run command."""
_runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"revert",
"--guidance",
"Use FastAPI instead",
"--dry-run",
"--plan",
_PLAN_ULID,
"--format",
"plain",
],
)
class M3FixtureLoadSuite:
"""Benchmark loading M3 fixture files."""
def time_load_decision_tree_fixture(self) -> None:
"""Benchmark loading decision_tree_outputs.json fixture."""
fixture_path = _FIXTURES_DIR / "decision_tree_outputs.json"
with open(fixture_path) as f:
json.load(f)
def time_load_validation_attachments_fixture(self) -> None:
"""Benchmark loading validation_attachments.json fixture."""
fixture_path = _FIXTURES_DIR / "validation_attachments.json"
with open(fixture_path) as f:
json.load(f)
def time_load_invariant_configs_fixture(self) -> None:
"""Benchmark loading invariant_configs.json fixture."""
fixture_path = _FIXTURES_DIR / "invariant_configs.json"
with open(fixture_path) as f:
json.load(f)
+111
View File
@@ -456,6 +456,59 @@ Performance baselines for persistence test operations:
Run with: `nox -s benchmark`
### ASV Benchmarks (`benchmarks/decision_persistence_bench.py`)
Performance baselines for decision persistence (serialization round-trip) operations:
- `DecisionDumpRoundTripSuite.time_root_dump` -- Root decision model_dump
- `DecisionDumpRoundTripSuite.time_root_validate` -- Root decision model_validate
- `DecisionDumpRoundTripSuite.time_root_roundtrip` -- Root decision full round-trip
- `DecisionDumpRoundTripSuite.time_child_roundtrip` -- Child decision full round-trip
- `DecisionDumpRoundTripSuite.time_full_roundtrip` -- Fully-populated decision round-trip
- `DecisionJsonRoundTripSuite.time_root_dump_json` -- Root decision model_dump_json
- `DecisionJsonRoundTripSuite.time_root_json_roundtrip` -- Root decision JSON round-trip
- `DecisionJsonRoundTripSuite.time_full_json_roundtrip` -- Full decision JSON round-trip
- `DecisionJsonRoundTripSuite.time_root_json_validate` -- Root decision model_validate from JSON
- `DecisionJsonRoundTripSuite.time_full_json_validate` -- Full decision model_validate from JSON
- `DecisionSnapshotPersistenceSuite.time_empty_snapshot_roundtrip` -- Empty snapshot round-trip
- `DecisionSnapshotPersistenceSuite.time_small_snapshot_roundtrip` -- Small snapshot round-trip
- `DecisionSnapshotPersistenceSuite.time_large_snapshot_roundtrip` -- Large snapshot (50 resources) round-trip
- `DecisionTreeSerializationSuite.time_small_tree_roundtrip` -- 3-node tree round-trip
- `DecisionTreeSerializationSuite.time_medium_tree_roundtrip` -- 7-node tree round-trip
- `DecisionTreeSerializationSuite.time_large_tree_roundtrip` -- 15-node tree round-trip
- `DecisionTreeSerializationSuite.time_large_tree_json_roundtrip` -- 15-node tree JSON round-trip
Run with: `nox -s benchmark`
### Behave: Decision Persistence (`features/decision_persistence.feature`)
21 scenarios covering Decision serialization round-trips, context-snapshot
persistence, correction-chain reconstruction, and decision-tree
reconstruction from serialized data:
- **CRUD round-trips**: Root and child decisions survive `model_dump` / `model_validate`.
- **JSON round-trips**: Decisions with artifacts survive `model_dump_json` / `model_validate`.
- **Snapshot persistence**: Empty snapshots, snapshots with resources, and actor state refs.
- **Correction chains**: Correction metadata, superseded_by, and 3-decision chain reconstruction.
- **Tree reconstruction**: 3-level tree (7 nodes) with parent-link integrity verification.
- **Edge cases**: Empty alternatives, confidence boundaries (0.0, 1.0, None), all 11 types, long rationale.
- **Downstream IDs**: `downstream_decision_ids` and `downstream_plan_ids` survive round-trip.
Step definitions: `features/steps/decision_persistence_steps.py`
### Robot Framework: Decision Persistence (`robot/decision_persistence.robot`)
7 integration smoke tests exercising decision persistence round-trips via
a helper script (`robot/helper_decision_persistence.py`):
- Root decision round-trip
- Child decision round-trip
- Context snapshot persistence
- JSON serialization round-trip
- Correction chain persistence
- Decision tree reconstruction
- All 11 decision types round-trip
### Behave: Skill Registry Persistence (`features/skill_registry.feature`)
23 scenarios covering `SkillRepository` and `SkillRegistryService` operations:
@@ -1090,3 +1143,61 @@ To perform a quick M1 source-code smoke run:
- **Coverage drops**: The M1 smoke tests cover fixture loading, CLI argument
parsing, and mock service integration. If coverage drops, look at
`build/htmlcov/index.html` for uncovered lines in plan/action CLI commands.
## M3 Decision Tree, Validation Gating, and Invariant Enforcement Smoke Tests
### Overview
The M3 smoke suite validates the decision tree, correction workflow,
validation gating (add / attach / detach), and invariant constraint
(add / list / remove) subsystems via CLI integration tests.
Issue: [#179](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/179)
### Files
| File | Purpose |
|------|------|
| `features/m3_decision_validation_smoke.feature` | Behave BDD scenarios (25 scenarios) |
| `features/steps/m3_decision_validation_smoke_steps.py` | Step implementations |
| `features/fixtures/m3/decision_tree_outputs.json` | Decision tree fixture data |
| `features/fixtures/m3/validation_attachments.json` | Validation attachment fixture data |
| `features/fixtures/m3/invariant_configs.json` | Invariant config fixture data |
| `robot/m3_decision_validation_smoke.robot` | Robot Framework integration tests (8 cases) |
| `robot/helper_m3_decision_validation_smoke.py` | Robot helper with 8 subcommands |
| `benchmarks/m3_smoke_bench.py` | ASV benchmark suites (5 classes) |
### Scenario Categories
1. **Decision tree fixture loading** -- Loads JSON fixtures and verifies
decision tree structure, invariant nodes, and correction metadata.
2. **Validation fixture loading** -- Verifies validation attachment data
including required and informational modes.
3. **Invariant fixture loading** -- Verifies invariant config data
including merge set precedence.
4. **Invariant CLI** -- Tests `agents invariant add`, `list`, and `remove`
commands with global and project scopes.
5. **Validation CLI** -- Tests `agents validation add`, `attach`, and
`detach` commands.
6. **Plan correct CLI** -- Tests `agents plan correct` with `--dry-run`,
`--mode revert`, and `--mode append`.
7. **Negative cases** -- Empty guidance, invalid correction mode.
### Mock Strategy
- **Invariant CLI** patches `cleveragents.cli.commands.invariant._get_service`
- **Validation CLI** patches `cleveragents.cli.commands.validation._get_tool_registry_service`
- **Plan CLI** patches `cleveragents.cli.commands.plan._get_lifecycle_service`
- **Correction** patches `CorrectionService` constructor at module level
### Failure Triage Tips
- **`AmbiguousStep` errors**: All M3 smoke steps are prefixed with `m3 smoke`.
If ambiguous, check that no other step file defines a conflicting pattern.
- **Fixture file not found**: Verify `features/fixtures/m3/` contains all
three JSON fixture files.
- **Robot `FAIL` sentinel**: Each Robot helper subcommand prints a detailed
`FAIL:` line with exit code and output. Check the Robot log.
- **Coverage drops**: The M3 smoke tests cover invariant/validation CLI
commands, plan correct workflow, and fixture loading. Check
`build/htmlcov/index.html` for uncovered lines.
@@ -0,0 +1,94 @@
{
"fixtures": [
{
"name": "minimal_decision_tree",
"description": "A minimal decision tree with root prompt and one strategy choice",
"decisions": [
{
"decision_type": "prompt_definition",
"sequence_number": 0,
"question": "What should we build?",
"chosen_option": "A REST API for user management",
"confidence_score": 0.95,
"is_root": true
},
{
"decision_type": "strategy_choice",
"sequence_number": 1,
"question": "Which framework to use?",
"chosen_option": "FastAPI",
"alternatives_considered": ["Flask", "Django"],
"confidence_score": 0.85,
"is_root": false
}
]
},
{
"name": "tree_with_invariant",
"description": "Decision tree with an invariant enforcement node",
"decisions": [
{
"decision_type": "prompt_definition",
"sequence_number": 0,
"question": "Plan the database migration",
"chosen_option": "Migrate user table to new schema",
"confidence_score": 0.90,
"is_root": true
},
{
"decision_type": "invariant_enforced",
"sequence_number": 1,
"question": "Should production data be preserved?",
"chosen_option": "Yes, all existing data must survive migration",
"confidence_score": 1.0,
"is_root": false,
"invariant_text": "Never delete production data"
},
{
"decision_type": "strategy_choice",
"sequence_number": 2,
"question": "Migration strategy?",
"chosen_option": "Blue-green deployment with rollback",
"alternatives_considered": ["In-place migration", "Shadow writes"],
"confidence_score": 0.80,
"is_root": false
}
]
},
{
"name": "tree_with_correction",
"description": "Decision tree that includes a correction decision superseding a prior choice",
"decisions": [
{
"decision_type": "prompt_definition",
"sequence_number": 0,
"question": "Implement authentication",
"chosen_option": "JWT-based auth with refresh tokens",
"confidence_score": 0.90,
"is_root": true
},
{
"decision_type": "strategy_choice",
"sequence_number": 1,
"question": "Token storage approach?",
"chosen_option": "HttpOnly cookies",
"alternatives_considered": ["localStorage", "sessionStorage"],
"confidence_score": 0.75,
"is_root": false,
"is_superseded": true
},
{
"decision_type": "strategy_choice",
"sequence_number": 2,
"question": "Token storage approach? (corrected)",
"chosen_option": "localStorage with XSS mitigation",
"alternatives_considered": ["HttpOnly cookies", "sessionStorage"],
"confidence_score": 0.82,
"is_root": false,
"is_correction": true,
"correction_reason": "HttpOnly cookies cause issues with SSR"
}
]
}
]
}
@@ -0,0 +1,69 @@
{
"fixtures": [
{
"name": "global_invariant",
"description": "A system-wide invariant constraint",
"invariant": {
"text": "Never delete production data",
"scope": "global",
"source_name": "system"
}
},
{
"name": "project_invariant",
"description": "A project-scoped invariant constraint",
"invariant": {
"text": "All API changes need tests",
"scope": "project",
"source_name": "local/m3-smoke-proj"
}
},
{
"name": "plan_invariant",
"description": "A plan-scoped invariant",
"invariant": {
"text": "Use only approved libraries",
"scope": "plan",
"source_name": "01M3SM0KE00000000000000001"
}
},
{
"name": "invariant_set_merge",
"description": "Invariants for merge precedence testing",
"global_invariants": [
{
"text": "Never delete production data",
"scope": "global",
"source_name": "system"
},
{
"text": "All changes must be reversible",
"scope": "global",
"source_name": "system"
}
],
"project_invariants": [
{
"text": "All API changes need tests",
"scope": "project",
"source_name": "local/m3-smoke-proj"
},
{
"text": "Never delete production data",
"scope": "project",
"source_name": "local/m3-smoke-proj"
}
],
"plan_invariants": [
{
"text": "Use only approved libraries",
"scope": "plan",
"source_name": "01M3SM0KE00000000000000001"
}
],
"expected_merged_count": 4,
"expected_dedup_text": "Never delete production data",
"expected_dedup_scope": "plan"
}
]
}
@@ -0,0 +1,47 @@
{
"fixtures": [
{
"name": "coverage_validation",
"description": "A coverage-check validation with required mode",
"validation": {
"name": "local/coverage-check",
"description": "Check code coverage meets threshold",
"source": "custom",
"mode": "required",
"code": "def run(inputs): return {'passed': inputs['coverage'] >= 80}"
}
},
{
"name": "lint_validation",
"description": "A lint-check validation with informational mode",
"validation": {
"name": "local/lint-check",
"description": "Check lint results",
"source": "custom",
"mode": "informational",
"code": "def run(inputs): return {'passed': inputs['warnings'] == 0}"
}
},
{
"name": "attachment_required",
"description": "A validation attachment in required mode",
"attachment": {
"validation_name": "local/coverage-check",
"resource_id": "git-checkout/my-repo",
"mode": "required",
"project_name": "local/m3-smoke-proj"
}
},
{
"name": "attachment_informational",
"description": "A validation attachment in informational mode",
"attachment": {
"validation_name": "local/lint-check",
"resource_id": "git-checkout/my-repo",
"mode": "informational",
"project_name": null,
"plan_id": "01M3SM0KE00000000000000001"
}
}
]
}
@@ -0,0 +1,136 @@
Feature: M3 decision tree, validation gating, and invariant enforcement smoke tests
As a developer working with the CleverAgents M3 milestone
I want to verify decision trees, corrections, validation lifecycle,
and invariant constraints work end-to-end via the CLI
So that the M3 foundation for decision + validation + invariant is solid
Background:
Given a m3 smoke test runner
And a m3 smoke mocked environment
# --- Decision tree fixture loading ---
Scenario: M3 smoke load decision tree fixture
When I m3 smoke load the decision tree fixture
Then the m3 smoke decision tree fixture should have a minimal tree entry
And the m3 smoke minimal tree should contain 2 decisions
Scenario: M3 smoke load decision tree with invariant node
When I m3 smoke load the decision tree fixture
Then the m3 smoke decision tree fixture should have an invariant tree entry
And the m3 smoke invariant tree should contain an invariant_enforced decision
Scenario: M3 smoke load decision tree with correction
When I m3 smoke load the decision tree fixture
Then the m3 smoke decision tree fixture should have a correction tree entry
And the m3 smoke correction tree should contain a superseded decision
And the m3 smoke correction tree should contain a correction decision
# --- Validation fixture loading ---
Scenario: M3 smoke load validation attachment fixtures
When I m3 smoke load the validation attachment fixture
Then the m3 smoke validation fixture should have a coverage entry
And the m3 smoke coverage validation should have mode "required"
Scenario: M3 smoke load validation attachment with informational mode
When I m3 smoke load the validation attachment fixture
Then the m3 smoke validation fixture should have a lint entry
And the m3 smoke lint validation should have mode "informational"
# --- Invariant fixture loading ---
Scenario: M3 smoke load invariant config fixtures
When I m3 smoke load the invariant config fixture
Then the m3 smoke invariant fixture should have a global entry
And the m3 smoke global invariant text should be "Never delete production data"
Scenario: M3 smoke load invariant merge set
When I m3 smoke load the invariant config fixture
Then the m3 smoke invariant fixture should have a merge set entry
And the m3 smoke merge set should define expected merged count 4
# --- Invariant CLI: add ---
Scenario: M3 smoke invariant add global
When I m3 smoke invoke invariant add with text "Never delete production data" and global scope
Then the m3 smoke invariant add should succeed
And the m3 smoke invariant output should contain "Never delete production data"
Scenario: M3 smoke invariant add project scoped
When I m3 smoke invoke invariant add with text "All API changes need tests" and project "local/m3-smoke-proj"
Then the m3 smoke invariant add should succeed
# --- Invariant CLI: list ---
Scenario: M3 smoke invariant list shows entries
Given m3 smoke invariants have been added
When I m3 smoke invoke invariant list
Then the m3 smoke invariant list should succeed
And the m3 smoke invariant list output should contain "Never delete production data"
Scenario: M3 smoke invariant list with project filter
Given m3 smoke invariants have been added
When I m3 smoke invoke invariant list with project filter "local/m3-smoke-proj"
Then the m3 smoke invariant list should succeed
# --- Invariant CLI: remove ---
Scenario: M3 smoke invariant remove by id
Given m3 smoke invariants have been added
When I m3 smoke invoke invariant remove with a known id
Then the m3 smoke invariant remove should succeed
Scenario: M3 smoke invariant remove unknown id fails
When I m3 smoke invoke invariant remove with id "01UNKNOWN000000000000000000"
Then the m3 smoke invariant remove should fail
# --- Validation CLI: add ---
Scenario: M3 smoke validation add from config
Given a m3 smoke temporary validation config file
When I m3 smoke invoke validation add with the config
Then the m3 smoke validation add should succeed
And the m3 smoke validation output should contain "coverage-check"
# --- Validation CLI: attach ---
Scenario: M3 smoke validation attach to resource
Given a m3 smoke validation "local/coverage-check" is registered
When I m3 smoke invoke validation attach "local/coverage-check" to "git-checkout/my-repo"
Then the m3 smoke validation attach should succeed
# --- Validation CLI: detach ---
Scenario: M3 smoke validation detach by id
Given a m3 smoke validation attachment exists
When I m3 smoke invoke validation detach with the attachment id
Then the m3 smoke validation detach should succeed
# --- Plan correct CLI ---
Scenario: M3 smoke plan correct dry run
Given a m3 smoke plan with decisions exists
When I m3 smoke invoke plan correct in dry-run mode
Then the m3 smoke plan correct dry run should succeed
And the m3 smoke correction output should contain "Dry Run"
Scenario: M3 smoke plan correct revert executes
Given a m3 smoke plan with decisions exists
When I m3 smoke invoke plan correct in revert mode
Then the m3 smoke plan correct should succeed
Scenario: M3 smoke plan correct append executes
Given a m3 smoke plan with decisions exists
When I m3 smoke invoke plan correct in append mode
Then the m3 smoke plan correct should succeed
# --- Negative cases ---
Scenario: M3 smoke plan correct with empty guidance fails
When I m3 smoke invoke plan correct with empty guidance
Then the m3 smoke plan correct should fail
Scenario: M3 smoke plan correct with invalid mode fails
When I m3 smoke invoke plan correct with invalid mode "rollback"
Then the m3 smoke plan correct should fail
@@ -0,0 +1,757 @@
"""Step definitions for M3 decision tree, validation, and invariant smoke tests.
All step names are prefixed with ``m3 smoke`` to avoid ``AmbiguousStep``
conflicts with existing steps.
"""
from __future__ import annotations
import contextlib
import json
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from cleveragents.cli.commands.invariant import app as invariant_app
from cleveragents.cli.commands.plan import app as plan_app
from cleveragents.cli.commands.validation import app as validation_app
from cleveragents.domain.models.core.correction import (
CorrectionMode,
CorrectionRequest,
CorrectionStatus,
)
from cleveragents.domain.models.core.invariant import Invariant, InvariantScope
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m3"
_PLAN_ULID = "01M3SM0KE00000000000000001"
_DECISION_ULID = "01M3DEC1S10N0000000000001"
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a m3 smoke test runner")
def step_m3_smoke_runner(context: Context) -> None:
"""Set up the CLI runner for M3 smoke tests."""
context.runner = CliRunner()
@given("a m3 smoke mocked environment")
def step_m3_smoke_mock_env(context: Context) -> None:
"""Set up the mocked services for M3 smoke tests."""
context.mock_invariant_service = MagicMock()
context.mock_tool_registry_service = MagicMock()
context.mock_correction_service = MagicMock()
context.mock_lifecycle_service = MagicMock()
context.invariant_patcher = patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=context.mock_invariant_service,
)
context.validation_patcher = patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=context.mock_tool_registry_service,
)
context.plan_patcher = patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=context.mock_lifecycle_service,
)
context.invariant_patcher.start()
context.validation_patcher.start()
context.plan_patcher.start()
context.last_result = None
context._m3_known_invariant_id = None
context._m3_known_attachment_id = None
# ---------------------------------------------------------------------------
# Decision tree fixture loading
# ---------------------------------------------------------------------------
@when("I m3 smoke load the decision tree fixture")
def step_m3_load_decision_tree(context: Context) -> None:
"""Load the decision tree outputs fixture JSON."""
fixture_path = _FIXTURES_DIR / "decision_tree_outputs.json"
with open(fixture_path) as f:
context.decision_tree_fixtures = json.load(f)
@then("the m3 smoke decision tree fixture should have a minimal tree entry")
def step_m3_has_minimal_tree(context: Context) -> None:
"""Verify fixture has a minimal_decision_tree entry."""
fixtures = context.decision_tree_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "minimal_decision_tree" in names, (
f"Expected 'minimal_decision_tree' in {names}"
)
@then("the m3 smoke minimal tree should contain {count:d} decisions")
def step_m3_minimal_tree_count(context: Context, count: int) -> None:
"""Verify the minimal tree has the expected number of decisions."""
fixtures = context.decision_tree_fixtures["fixtures"]
minimal = next(f for f in fixtures if f["name"] == "minimal_decision_tree")
actual = len(minimal["decisions"])
assert actual == count, f"Expected {count} decisions, got {actual}"
@then("the m3 smoke decision tree fixture should have an invariant tree entry")
def step_m3_has_invariant_tree(context: Context) -> None:
"""Verify fixture has a tree_with_invariant entry."""
fixtures = context.decision_tree_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "tree_with_invariant" in names, f"Expected 'tree_with_invariant' in {names}"
@then("the m3 smoke invariant tree should contain an invariant_enforced decision")
def step_m3_invariant_tree_has_enforced(context: Context) -> None:
"""Verify the invariant tree has an invariant_enforced decision."""
fixtures = context.decision_tree_fixtures["fixtures"]
tree = next(f for f in fixtures if f["name"] == "tree_with_invariant")
types = [d["decision_type"] for d in tree["decisions"]]
assert "invariant_enforced" in types, f"Expected 'invariant_enforced' in {types}"
@then("the m3 smoke decision tree fixture should have a correction tree entry")
def step_m3_has_correction_tree(context: Context) -> None:
"""Verify fixture has a tree_with_correction entry."""
fixtures = context.decision_tree_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "tree_with_correction" in names, (
f"Expected 'tree_with_correction' in {names}"
)
@then("the m3 smoke correction tree should contain a superseded decision")
def step_m3_correction_tree_has_superseded(context: Context) -> None:
"""Verify the correction tree has a superseded decision."""
fixtures = context.decision_tree_fixtures["fixtures"]
tree = next(f for f in fixtures if f["name"] == "tree_with_correction")
superseded = [d for d in tree["decisions"] if d.get("is_superseded")]
assert len(superseded) > 0, "Expected at least one superseded decision"
@then("the m3 smoke correction tree should contain a correction decision")
def step_m3_correction_tree_has_correction(context: Context) -> None:
"""Verify the correction tree has a correction decision."""
fixtures = context.decision_tree_fixtures["fixtures"]
tree = next(f for f in fixtures if f["name"] == "tree_with_correction")
corrections = [d for d in tree["decisions"] if d.get("is_correction")]
assert len(corrections) > 0, "Expected at least one correction decision"
# ---------------------------------------------------------------------------
# Validation fixture loading
# ---------------------------------------------------------------------------
@when("I m3 smoke load the validation attachment fixture")
def step_m3_load_validation_fixtures(context: Context) -> None:
"""Load the validation attachments fixture JSON."""
fixture_path = _FIXTURES_DIR / "validation_attachments.json"
with open(fixture_path) as f:
context.validation_fixtures = json.load(f)
@then("the m3 smoke validation fixture should have a coverage entry")
def step_m3_has_coverage_validation(context: Context) -> None:
"""Verify fixture has a coverage_validation entry."""
fixtures = context.validation_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "coverage_validation" in names, f"Expected 'coverage_validation' in {names}"
@then('the m3 smoke coverage validation should have mode "{mode}"')
def step_m3_coverage_mode(context: Context, mode: str) -> None:
"""Verify coverage validation has expected mode."""
fixtures = context.validation_fixtures["fixtures"]
entry = next(f for f in fixtures if f["name"] == "coverage_validation")
actual = entry["validation"]["mode"]
assert actual == mode, f"Expected mode '{mode}', got '{actual}'"
@then("the m3 smoke validation fixture should have a lint entry")
def step_m3_has_lint_validation(context: Context) -> None:
"""Verify fixture has a lint_validation entry."""
fixtures = context.validation_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "lint_validation" in names, f"Expected 'lint_validation' in {names}"
@then('the m3 smoke lint validation should have mode "{mode}"')
def step_m3_lint_mode(context: Context, mode: str) -> None:
"""Verify lint validation has expected mode."""
fixtures = context.validation_fixtures["fixtures"]
entry = next(f for f in fixtures if f["name"] == "lint_validation")
actual = entry["validation"]["mode"]
assert actual == mode, f"Expected mode '{mode}', got '{actual}'"
# ---------------------------------------------------------------------------
# Invariant fixture loading
# ---------------------------------------------------------------------------
@when("I m3 smoke load the invariant config fixture")
def step_m3_load_invariant_fixtures(context: Context) -> None:
"""Load the invariant configs fixture JSON."""
fixture_path = _FIXTURES_DIR / "invariant_configs.json"
with open(fixture_path) as f:
context.invariant_fixtures = json.load(f)
@then("the m3 smoke invariant fixture should have a global entry")
def step_m3_has_global_invariant(context: Context) -> None:
"""Verify fixture has a global_invariant entry."""
fixtures = context.invariant_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "global_invariant" in names, f"Expected 'global_invariant' in {names}"
@then('the m3 smoke global invariant text should be "{text}"')
def step_m3_global_invariant_text(context: Context, text: str) -> None:
"""Verify global invariant has expected text."""
fixtures = context.invariant_fixtures["fixtures"]
entry = next(f for f in fixtures if f["name"] == "global_invariant")
actual = entry["invariant"]["text"]
assert actual == text, f"Expected text '{text}', got '{actual}'"
@then("the m3 smoke invariant fixture should have a merge set entry")
def step_m3_has_merge_set(context: Context) -> None:
"""Verify fixture has an invariant_set_merge entry."""
fixtures = context.invariant_fixtures["fixtures"]
names = [f["name"] for f in fixtures]
assert "invariant_set_merge" in names, f"Expected 'invariant_set_merge' in {names}"
@then("the m3 smoke merge set should define expected merged count {count:d}")
def step_m3_merge_set_count(context: Context, count: int) -> None:
"""Verify merge set defines expected merged count."""
fixtures = context.invariant_fixtures["fixtures"]
entry = next(f for f in fixtures if f["name"] == "invariant_set_merge")
actual = entry["expected_merged_count"]
assert actual == count, f"Expected merged count {count}, got {actual}"
# ---------------------------------------------------------------------------
# Invariant CLI: add
# ---------------------------------------------------------------------------
def _make_m3_invariant(
*,
text: str = "Never delete production data",
scope: InvariantScope = InvariantScope.GLOBAL,
source_name: str = "system",
) -> Invariant:
"""Create an Invariant instance for M3 smoke tests."""
return Invariant(
id=_INVARIANT_ULID,
text=text,
scope=scope,
source_name=source_name,
)
@when('I m3 smoke invoke invariant add with text "{text}" and global scope')
def step_m3_invariant_add_global(context: Context, text: str) -> None:
"""Invoke invariant add with global scope."""
context.mock_invariant_service.add_invariant.return_value = _make_m3_invariant(
text=text
)
result = context.runner.invoke(
invariant_app,
["add", "--global", text, "--format", "plain"],
)
context.last_result = result
@when('I m3 smoke invoke invariant add with text "{text}" and project "{project}"')
def step_m3_invariant_add_project(context: Context, text: str, project: str) -> None:
"""Invoke invariant add with project scope."""
context.mock_invariant_service.add_invariant.return_value = _make_m3_invariant(
text=text,
scope=InvariantScope.PROJECT,
source_name=project,
)
result = context.runner.invoke(
invariant_app,
["add", "--project", project, text, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke invariant add should succeed")
def step_m3_invariant_add_ok(context: Context) -> None:
"""Verify invariant add succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
@then('the m3 smoke invariant output should contain "{text}"')
def step_m3_invariant_output_contains(context: Context, text: str) -> None:
"""Verify invariant output contains expected text."""
output = context.last_result.output
assert text in output, f"Expected '{text}' in: {output}"
# ---------------------------------------------------------------------------
# Invariant CLI: list
# ---------------------------------------------------------------------------
@given("m3 smoke invariants have been added")
def step_m3_invariants_added(context: Context) -> None:
"""Set up mock invariant service with pre-added invariants."""
inv1 = _make_m3_invariant()
inv2 = _make_m3_invariant(
text="All API changes need tests",
scope=InvariantScope.PROJECT,
source_name="local/m3-smoke-proj",
)
context.mock_invariant_service.list_invariants.return_value = [inv1, inv2]
context._m3_known_invariant_id = _INVARIANT_ULID
@when("I m3 smoke invoke invariant list")
def step_m3_invariant_list(context: Context) -> None:
"""Invoke invariant list."""
result = context.runner.invoke(
invariant_app,
["list", "--format", "plain"],
)
context.last_result = result
@when('I m3 smoke invoke invariant list with project filter "{project}"')
def step_m3_invariant_list_project(context: Context, project: str) -> None:
"""Invoke invariant list with project filter."""
result = context.runner.invoke(
invariant_app,
["list", "--project", project, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke invariant list should succeed")
def step_m3_invariant_list_ok(context: Context) -> None:
"""Verify invariant list succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
@then('the m3 smoke invariant list output should contain "{text}"')
def step_m3_invariant_list_contains(context: Context, text: str) -> None:
"""Verify invariant list output contains expected text."""
output = context.last_result.output
assert text in output, f"Expected '{text}' in: {output}"
# ---------------------------------------------------------------------------
# Invariant CLI: remove
# ---------------------------------------------------------------------------
@when("I m3 smoke invoke invariant remove with a known id")
def step_m3_invariant_remove_known(context: Context) -> None:
"""Invoke invariant remove with a known invariant ID."""
inv_id = context._m3_known_invariant_id or _INVARIANT_ULID
context.mock_invariant_service.remove_invariant.return_value = _make_m3_invariant()
result = context.runner.invoke(
invariant_app,
["remove", "--yes", inv_id, "--format", "plain"],
)
context.last_result = result
@when('I m3 smoke invoke invariant remove with id "{inv_id}"')
def step_m3_invariant_remove_by_id(context: Context, inv_id: str) -> None:
"""Invoke invariant remove with specific id."""
from cleveragents.core.exceptions import ResourceNotFoundError
context.mock_invariant_service.remove_invariant.side_effect = ResourceNotFoundError(
message=f"Invariant not found: {inv_id}",
resource_type="invariant",
resource_id=inv_id,
)
result = context.runner.invoke(
invariant_app,
["remove", "--yes", inv_id, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke invariant remove should succeed")
def step_m3_invariant_remove_ok(context: Context) -> None:
"""Verify invariant remove succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
@then("the m3 smoke invariant remove should fail")
def step_m3_invariant_remove_fail(context: Context) -> None:
"""Verify invariant remove failed."""
assert context.last_result is not None
assert context.last_result.exit_code != 0
# ---------------------------------------------------------------------------
# Validation CLI: add
# ---------------------------------------------------------------------------
@given("a m3 smoke temporary validation config file")
def step_m3_temp_validation_config(context: Context) -> None:
"""Create a temp YAML config file for validation add."""
config_content = (
"name: local/coverage-check\n"
"description: Check code coverage meets threshold\n"
"source: custom\n"
"mode: required\n"
"code: |\n"
" def run(inputs):\n"
" return {'passed': inputs['coverage'] >= 80}\n"
)
with tempfile.NamedTemporaryFile(
mode="w",
suffix=".yaml",
delete=False,
) as tmp:
tmp.write(config_content)
context.m3_validation_config = tmp.name
mock_validation = MagicMock()
mock_validation.name = "local/coverage-check"
mock_validation.as_cli_dict.return_value = {
"name": "local/coverage-check",
"description": "Check code coverage meets threshold",
"source": "custom",
"mode": "required",
}
context.mock_tool_registry_service.register_tool.return_value = mock_validation
@when("I m3 smoke invoke validation add with the config")
def step_m3_validation_add(context: Context) -> None:
"""Invoke validation add CLI with config file."""
result = context.runner.invoke(
validation_app,
["add", "--config", context.m3_validation_config, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke validation add should succeed")
def step_m3_validation_add_ok(context: Context) -> None:
"""Verify validation add succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
@then('the m3 smoke validation output should contain "{text}"')
def step_m3_validation_output_contains(context: Context, text: str) -> None:
"""Verify validation output contains expected text."""
output = context.last_result.output
assert text in output, f"Expected '{text}' in: {output}"
# ---------------------------------------------------------------------------
# Validation CLI: attach
# ---------------------------------------------------------------------------
@given('a m3 smoke validation "{name}" is registered')
def step_m3_validation_registered(context: Context, name: str) -> None:
"""Set up a registered validation for attach tests."""
mock_attachment = MagicMock()
mock_attachment.attachment_id = _ATTACHMENT_ULID
mock_attachment.validation_name = name
mock_attachment.resource_id = "git-checkout/my-repo"
mock_attachment.mode = "required"
mock_attachment.project_name = None
mock_attachment.plan_id = None
mock_attachment.created_at = "2026-01-01T00:00:00"
context.mock_tool_registry_service.attach_validation.return_value = mock_attachment
context._m3_known_attachment_id = _ATTACHMENT_ULID
@when('I m3 smoke invoke validation attach "{name}" to "{resource}"')
def step_m3_validation_attach(context: Context, name: str, resource: str) -> None:
"""Invoke validation attach CLI."""
result = context.runner.invoke(
validation_app,
["attach", resource, name, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke validation attach should succeed")
def step_m3_validation_attach_ok(context: Context) -> None:
"""Verify validation attach succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
# ---------------------------------------------------------------------------
# Validation CLI: detach
# ---------------------------------------------------------------------------
@given("a m3 smoke validation attachment exists")
def step_m3_attachment_exists(context: Context) -> None:
"""Set up an existing validation attachment for detach tests."""
context.mock_tool_registry_service.detach_validation.return_value = True
context._m3_known_attachment_id = _ATTACHMENT_ULID
@when("I m3 smoke invoke validation detach with the attachment id")
def step_m3_validation_detach(context: Context) -> None:
"""Invoke validation detach CLI."""
att_id = context._m3_known_attachment_id or _ATTACHMENT_ULID
result = context.runner.invoke(
validation_app,
["detach", "--yes", att_id, "--format", "plain"],
)
context.last_result = result
@then("the m3 smoke validation detach should succeed")
def step_m3_validation_detach_ok(context: Context) -> None:
"""Verify validation detach succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
# ---------------------------------------------------------------------------
# Plan correct CLI
# ---------------------------------------------------------------------------
@given("a m3 smoke plan with decisions exists")
def step_m3_plan_with_decisions(context: Context) -> None:
"""Set up mock correction service for plan correct tests."""
mock_request = MagicMock(spec=CorrectionRequest)
mock_request.correction_id = _CORRECTION_ULID
mock_request.mode = CorrectionMode.REVERT
mock_request.target_decision_id = _DECISION_ULID
mock_request.guidance = "Use FastAPI instead"
mock_request.status = CorrectionStatus.PENDING
mock_impact = MagicMock()
mock_impact.affected_decisions = [_DECISION_ULID]
mock_impact.affected_files = ["src/main.py"]
mock_impact.estimated_cost = "low"
mock_impact.risk_level = "low"
mock_result = MagicMock()
mock_result.correction_id = _CORRECTION_ULID
mock_result.status = CorrectionStatus.APPLIED
mock_result.new_decisions = ["01M3NEWDEC000000000000001"]
mock_result.reverted_decisions = [_DECISION_ULID]
context.mock_correction_service.request_correction.return_value = mock_request
context.mock_correction_service.analyze_impact.return_value = mock_impact
context.mock_correction_service.execute_correction.return_value = mock_result
context.correction_patcher = patch(
"cleveragents.application.services.correction_service.CorrectionService",
return_value=context.mock_correction_service,
)
context.correction_patcher.start()
@when("I m3 smoke invoke plan correct in dry-run mode")
def step_m3_plan_correct_dry_run(context: Context) -> None:
"""Invoke plan correct with --dry-run."""
result = context.runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"revert",
"--guidance",
"Use FastAPI instead",
"--dry-run",
"--plan",
_PLAN_ULID,
"--format",
"plain",
],
)
context.last_result = result
@when("I m3 smoke invoke plan correct in revert mode")
def step_m3_plan_correct_revert(context: Context) -> None:
"""Invoke plan correct with revert mode."""
result = context.runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"revert",
"--guidance",
"Use FastAPI instead",
"--yes",
"--plan",
_PLAN_ULID,
"--format",
"plain",
],
)
context.last_result = result
@when("I m3 smoke invoke plan correct in append mode")
def step_m3_plan_correct_append(context: Context) -> None:
"""Invoke plan correct with append mode."""
context.mock_correction_service.request_correction.return_value.mode = (
CorrectionMode.APPEND
)
result = context.runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"append",
"--guidance",
"Add caching layer",
"--yes",
"--plan",
_PLAN_ULID,
"--format",
"plain",
],
)
context.last_result = result
@then("the m3 smoke plan correct dry run should succeed")
def step_m3_correct_dry_run_ok(context: Context) -> None:
"""Verify plan correct dry run succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
@then('the m3 smoke correction output should contain "{text}"')
def step_m3_correction_output_contains(context: Context, text: str) -> None:
"""Verify correction output contains expected text."""
output = context.last_result.output
assert text in output, f"Expected '{text}' in: {output}"
@then("the m3 smoke plan correct should succeed")
def step_m3_correct_ok(context: Context) -> None:
"""Verify plan correct succeeded."""
assert context.last_result is not None
assert context.last_result.exit_code == 0, (
f"Expected exit 0, got {context.last_result.exit_code}. "
f"Output: {context.last_result.output}"
)
# ---------------------------------------------------------------------------
# Negative cases
# ---------------------------------------------------------------------------
@when("I m3 smoke invoke plan correct with empty guidance")
def step_m3_correct_empty_guidance(context: Context) -> None:
"""Invoke plan correct with empty guidance string."""
result = context.runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"revert",
"--guidance",
"",
"--yes",
"--plan",
_PLAN_ULID,
],
)
context.last_result = result
@when('I m3 smoke invoke plan correct with invalid mode "{mode}"')
def step_m3_correct_invalid_mode(context: Context, mode: str) -> None:
"""Invoke plan correct with an invalid mode."""
result = context.runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
mode,
"--guidance",
"Some guidance",
"--yes",
"--plan",
_PLAN_ULID,
],
)
context.last_result = result
@then("the m3 smoke plan correct should fail")
def step_m3_correct_fail(context: Context) -> None:
"""Verify plan correct failed."""
assert context.last_result is not None
assert context.last_result.exit_code != 0
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
def after_scenario(context: Context, scenario: object) -> None:
"""Clean up patchers after each scenario."""
for name in (
"invariant_patcher",
"validation_patcher",
"plan_patcher",
"correction_patcher",
):
patcher = getattr(context, name, None)
if patcher:
with contextlib.suppress(RuntimeError):
patcher.stop()
@@ -0,0 +1,318 @@
"""Helper script for m3_decision_validation_smoke.robot E2E tests.
Each subcommand is a self-contained check that prints a sentinel on success.
Uses ``--format plain`` where possible to stabilise assertion strings.
"""
from __future__ import annotations
import sys
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
# Ensure local source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.invariant import app as invariant_app # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.cli.commands.validation import app as validation_app # noqa: E402
from cleveragents.domain.models.core.correction import ( # noqa: E402
CorrectionMode,
CorrectionStatus,
)
from cleveragents.domain.models.core.invariant import ( # noqa: E402
Invariant,
InvariantScope,
)
runner = CliRunner()
_PLAN_ULID = "01M3SM0KE00000000000000001"
_DECISION_ULID = "01M3DEC1S10N0000000000001"
_CORRECTION_ULID = "01M3C0RRECT10N00000000001"
_ATTACHMENT_ULID = "01M3ATTACH0000000000000001"
_INVARIANT_ULID = "01M3INVAR1ANT0000000000001"
def _mock_invariant(
*,
text: str = "Never delete production data",
scope: InvariantScope = InvariantScope.GLOBAL,
source_name: str = "system",
) -> Invariant:
"""Create an Invariant for M3 smoke tests."""
return Invariant(
id=_INVARIANT_ULID,
text=text,
scope=scope,
source_name=source_name,
)
# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------
def invariant_add_global() -> None:
"""Add a global invariant constraint."""
mock_svc = MagicMock()
mock_svc.add_invariant.return_value = _mock_invariant()
with patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=mock_svc,
):
result = runner.invoke(
invariant_app,
["add", "--global", "Never delete production data", "--format", "plain"],
)
if result.exit_code == 0:
print("m3-invariant-add-global-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def invariant_add_project() -> None:
"""Add a project-scoped invariant constraint."""
mock_svc = MagicMock()
mock_svc.add_invariant.return_value = _mock_invariant(
text="All API changes need tests",
scope=InvariantScope.PROJECT,
source_name="local/m3-smoke-proj",
)
with patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=mock_svc,
):
result = runner.invoke(
invariant_app,
[
"add",
"--project",
"local/m3-smoke-proj",
"All API changes need tests",
"--format",
"plain",
],
)
if result.exit_code == 0:
print("m3-invariant-add-project-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def invariant_list() -> None:
"""List invariants."""
mock_svc = MagicMock()
mock_svc.list_invariants.return_value = [
_mock_invariant(),
_mock_invariant(
text="All API changes need tests",
scope=InvariantScope.PROJECT,
source_name="local/m3-smoke-proj",
),
]
with patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=mock_svc,
):
result = runner.invoke(
invariant_app,
["list", "--format", "plain"],
)
if result.exit_code == 0:
print("m3-invariant-list-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def invariant_remove() -> None:
"""Remove an invariant by ID."""
mock_svc = MagicMock()
mock_svc.remove_invariant.return_value = _mock_invariant()
with patch(
"cleveragents.cli.commands.invariant._get_service",
return_value=mock_svc,
):
result = runner.invoke(
invariant_app,
["remove", "--yes", _INVARIANT_ULID, "--format", "plain"],
)
if result.exit_code == 0:
print("m3-invariant-remove-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def validation_add() -> None:
"""Register a validation from YAML config."""
config_content = (
"name: local/coverage-check\n"
"description: Check code coverage meets threshold\n"
"source: custom\n"
"mode: required\n"
"code: |\n"
" def run(inputs):\n"
" return {'passed': inputs['coverage'] >= 80}\n"
)
with tempfile.NamedTemporaryFile(
mode="w",
suffix=".yaml",
delete=False,
) as tmp:
tmp.write(config_content)
tmp_name = tmp.name
mock_svc = MagicMock()
mock_validation = MagicMock()
mock_validation.name = "local/coverage-check"
mock_validation.as_cli_dict.return_value = {
"name": "local/coverage-check",
"description": "Check code coverage meets threshold",
"source": "custom",
"mode": "required",
}
mock_svc.register_tool.return_value = mock_validation
with patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=mock_svc,
):
result = runner.invoke(
validation_app,
["add", "--config", tmp_name, "--format", "plain"],
)
if result.exit_code == 0 and "coverage-check" in result.output:
print("m3-validation-add-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def validation_attach() -> None:
"""Attach a validation to a resource."""
mock_svc = MagicMock()
mock_attachment = MagicMock()
mock_attachment.attachment_id = _ATTACHMENT_ULID
mock_attachment.validation_name = "local/coverage-check"
mock_attachment.resource_id = "git-checkout/my-repo"
mock_attachment.mode = "required"
mock_attachment.project_name = None
mock_attachment.plan_id = None
mock_attachment.created_at = "2026-01-01T00:00:00"
mock_svc.attach_validation.return_value = mock_attachment
with patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=mock_svc,
):
result = runner.invoke(
validation_app,
[
"attach",
"git-checkout/my-repo",
"local/coverage-check",
"--format",
"plain",
],
)
if result.exit_code == 0:
print("m3-validation-attach-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def validation_detach() -> None:
"""Detach a validation from a resource."""
mock_svc = MagicMock()
mock_svc.detach_validation.return_value = True
with patch(
"cleveragents.cli.commands.validation._get_tool_registry_service",
return_value=mock_svc,
):
result = runner.invoke(
validation_app,
["detach", "--yes", _ATTACHMENT_ULID, "--format", "plain"],
)
if result.exit_code == 0:
print("m3-validation-detach-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
def plan_correct_dry_run() -> None:
"""Run plan correct in dry-run mode."""
mock_correction_svc = MagicMock()
mock_request = MagicMock()
mock_request.correction_id = _CORRECTION_ULID
mock_request.mode = CorrectionMode.REVERT
mock_request.target_decision_id = _DECISION_ULID
mock_request.guidance = "Use FastAPI instead"
mock_request.status = CorrectionStatus.PENDING
mock_impact = MagicMock()
mock_impact.affected_decisions = [_DECISION_ULID]
mock_impact.affected_files = ["src/main.py"]
mock_impact.estimated_cost = "low"
mock_impact.risk_level = "low"
mock_correction_svc.request_correction.return_value = mock_request
mock_correction_svc.analyze_impact.return_value = mock_impact
with patch(
"cleveragents.application.services.correction_service.CorrectionService",
return_value=mock_correction_svc,
):
result = runner.invoke(
plan_app,
[
"correct",
_DECISION_ULID,
"--mode",
"revert",
"--guidance",
"Use FastAPI instead",
"--dry-run",
"--plan",
_PLAN_ULID,
"--format",
"plain",
],
)
if result.exit_code == 0:
print("m3-plan-correct-dry-run-ok")
else:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------
_COMMANDS: dict[str, object] = {
"invariant-add-global": invariant_add_global,
"invariant-add-project": invariant_add_project,
"invariant-list": invariant_list,
"invariant-remove": invariant_remove,
"validation-add": validation_add,
"validation-attach": validation_attach,
"validation-detach": validation_detach,
"plan-correct-dry-run": plan_correct_dry_run,
}
if __name__ == "__main__":
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
sys.exit(1)
fn = _COMMANDS[sys.argv[1]]
fn() # type: ignore[operator]
+73
View File
@@ -0,0 +1,73 @@
*** Settings ***
Documentation M3 decision tree, validation gating, and invariant enforcement E2E smoke tests via CLI
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Variables ***
${HELPER} ${CURDIR}/helper_m3_decision_validation_smoke.py
*** Test Cases ***
M3 Invariant Add Global
[Documentation] Add a global invariant constraint via CLI
${result}= Run Process ${PYTHON} ${HELPER} invariant-add-global cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-invariant-add-global-ok
M3 Invariant Add Project Scoped
[Documentation] Add a project-scoped invariant constraint via CLI
${result}= Run Process ${PYTHON} ${HELPER} invariant-add-project cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-invariant-add-project-ok
M3 Invariant List
[Documentation] List invariants via CLI
${result}= Run Process ${PYTHON} ${HELPER} invariant-list cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-invariant-list-ok
M3 Invariant Remove
[Documentation] Remove an invariant by ID via CLI
${result}= Run Process ${PYTHON} ${HELPER} invariant-remove cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-invariant-remove-ok
M3 Validation Add From Config
[Documentation] Register a validation from YAML config via CLI
${result}= Run Process ${PYTHON} ${HELPER} validation-add cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-validation-add-ok
M3 Validation Attach
[Documentation] Attach a validation to a resource via CLI
${result}= Run Process ${PYTHON} ${HELPER} validation-attach cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-validation-attach-ok
M3 Validation Detach
[Documentation] Detach a validation from a resource via CLI
${result}= Run Process ${PYTHON} ${HELPER} validation-detach cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-validation-detach-ok
M3 Plan Correct Dry Run
[Documentation] Run plan correct in dry-run mode to preview correction impact
${result}= Run Process ${PYTHON} ${HELPER} plan-correct-dry-run cwd=${WORKSPACE} timeout=30s
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} m3-plan-correct-dry-run-ok