Compare commits

...

1 Commits

Author SHA1 Message Date
CoreRasurae 4b34758477 feat(apply): run validation attachments 2026-02-23 22:47:35 +00:00
12 changed files with 1666 additions and 30 deletions
+164
View File
@@ -0,0 +1,164 @@
"""ASV benchmarks for validation apply gate throughput.
Measures the performance of:
- ValidationAttachment creation
- DefaultValidationRunner execution
- ApplyValidationGate.run() with multiple attachments
- ApplyValidationSummary.to_plan_metadata() serialization
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
# Ensure the local *source* tree is importable.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.application.services.validation_apply import ( # noqa: E402
ApplyValidationGate,
ApplyValidationResult,
ApplyValidationSummary,
DefaultValidationRunner,
ValidationAttachment,
)
from cleveragents.domain.models.core.tool import ValidationMode # noqa: E402
# ---------------------------------------------------------------------------
# Benchmark: ValidationAttachment creation
# ---------------------------------------------------------------------------
class AttachmentCreation:
"""Benchmark suite for ``ValidationAttachment`` model creation."""
def time_create_required(self) -> None:
"""Create a required validation attachment."""
ValidationAttachment(
attachment_id="01ATT000000000000000000000",
validation_name="lint-check",
resource_id="res-001",
mode=ValidationMode.REQUIRED,
)
def time_create_informational(self) -> None:
"""Create an informational validation attachment."""
ValidationAttachment(
attachment_id="01ATT000000000000000000001",
validation_name="style-check",
resource_id="res-002",
mode=ValidationMode.INFORMATIONAL,
)
# ---------------------------------------------------------------------------
# Benchmark: DefaultValidationRunner
# ---------------------------------------------------------------------------
_ATTACHMENT = ValidationAttachment(
attachment_id="01ATT000000000000000000000",
validation_name="lint-check",
resource_id="res-001",
mode=ValidationMode.REQUIRED,
)
_CTX_PASS = {"lint-check": "passed"}
_CTX_FAIL = {"status": "no matches"}
class RunnerExecution:
"""Benchmark suite for ``DefaultValidationRunner``."""
def setup(self) -> None:
"""Prepare runner."""
self.runner = DefaultValidationRunner()
def time_run_pass(self) -> None:
"""Run validation that passes."""
self.runner.run_validation(_ATTACHMENT, _CTX_PASS)
def time_run_fail(self) -> None:
"""Run validation that fails."""
self.runner.run_validation(_ATTACHMENT, _CTX_FAIL)
# ---------------------------------------------------------------------------
# Benchmark: ApplyValidationGate
# ---------------------------------------------------------------------------
def _make_attachments(n: int) -> list[ValidationAttachment]:
"""Create N test attachments."""
return [
ValidationAttachment(
attachment_id=f"01ATT{i:021d}",
validation_name=f"val-{i}",
resource_id=f"res-{i}",
mode=ValidationMode.REQUIRED,
)
for i in range(n)
]
class GateExecution:
"""Benchmark suite for ``ApplyValidationGate.run``."""
def setup(self) -> None:
"""Prepare gate and attachments."""
self.runner = DefaultValidationRunner()
self.gate = ApplyValidationGate(runner=self.runner)
self.attachments_5 = _make_attachments(5)
self.attachments_20 = _make_attachments(20)
self.context = {f"val-{i}": "ok" for i in range(20)}
def time_gate_5_attachments(self) -> None:
"""Run gate with 5 attachments."""
self.gate.run("01PLAN0001", self.attachments_5, self.context)
def time_gate_20_attachments(self) -> None:
"""Run gate with 20 attachments."""
self.gate.run("01PLAN0001", self.attachments_20, self.context)
# ---------------------------------------------------------------------------
# Benchmark: Summary serialization
# ---------------------------------------------------------------------------
class SummarySerialization:
"""Benchmark suite for ``ApplyValidationSummary`` serialization."""
def setup(self) -> None:
"""Prepare summary with mixed results."""
results = []
for i in range(10):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{i:021d}",
validation_name=f"val-{i}",
resource_id=f"res-{i}",
mode=ValidationMode.REQUIRED,
passed=i % 3 != 0,
message="ok" if i % 3 != 0 else "failed",
)
)
self.summary = ApplyValidationSummary(
plan_id="01PLAN0001",
results=results,
)
def time_to_plan_metadata(self) -> None:
"""Serialize summary to plan metadata dict."""
self.summary.to_plan_metadata()
def time_format_cli_output(self) -> None:
"""Format summary for CLI display."""
self.summary.format_cli_output()
+78
View File
@@ -456,6 +456,57 @@ Performance baselines for persistence test operations:
Run with: `nox -s benchmark`
### Behave: Skill Registry Persistence (`features/skill_registry.feature`)
23 scenarios covering `SkillRepository` and `SkillRegistryService` operations:
- **Register**: Persist skills with each item type (`tool_ref`, `include`, `inline_tool`, `mcp_source`, `agent_source`), verify round-trip.
- **Duplicate rejection**: Second registration with the same name raises `DuplicateSkillError`.
- **Retrieve / list**: Get by name, list all, list with namespace filter.
- **Update / remove**: Change description, remove with cascade, non-existent skill error paths.
- **Overrides round-trip**: Override metadata survives persistence.
- **Ordering stability**: Mixed item types maintain stable `item_order` across round-trips.
- **Name validation**: Empty, no-slash, and special-character names rejected at persistence layer.
- **Large payload**: 50 tool refs persist correctly.
- **Domain objects**: Listed results are `Skill` domain instances.
Step definitions: `features/steps/skill_registry_steps.py`
**Fixtures**: Each scenario gets a fresh in-memory SQLite database. A single
shared `Session` is reused across all repository calls within each scenario via a
lambda session factory (`lambda: shared_session`). This mirrors the production
`UnitOfWork` pattern (ADR-019) and avoids transaction-scoping issues that arise
when multiple ephemeral sessions share one in-memory SQLite connection
(`StaticPool`): abandoned sessions can be garbage-collected at unpredictable
times, rolling back previously flushed but uncommitted data.
### Robot: Skill Registry (`robot/skill_registry.robot`)
5 smoke tests exercising the registry through a Python helper script:
| Test Case | Description |
|-----------|-------------|
| Register And Retrieve A Skill | Round-trip: register then get by name |
| List Skills With Namespace Filter | Register multiple skills, filter by namespace |
| Update A Skill | Change description after registration |
| Reject Duplicate Skill Name | Duplicate name produces error |
| Remove A Skill | Remove and verify absence |
Helper script: `robot/helper_skill_registry.py`
### ASV Benchmarks: Skill Registry (`benchmarks/skill_registry_bench.py`)
Performance baselines for skill registry persistence operations:
- `SkillModelConstruction.time_from_domain` -- Domain-to-ORM mapping
- `SkillModelReconstruction.time_to_domain` -- ORM-to-domain reconstruction
- `SkillRepositoryCRUD.time_create_skill` -- Single skill insert + commit
- `SkillRepositoryCRUD.time_get_skill` -- Lookup by name
- `SkillRepositoryCRUD.time_list_all` -- List all skills
- `SkillRepositoryCRUD.time_list_namespace` -- List with namespace filter
- `SkillRegistryListPerformance.time_list_100_skills` -- List with 100 seeded skills
- `SkillRegistryListPerformance.time_list_100_skills_filtered` -- Filtered list with 100 skills
### Running Persistence Tests
```bash
@@ -468,6 +519,9 @@ nox -s unit_tests -- features/plan_persistence.feature
# Run only action persistence scenarios
nox -s unit_tests -- features/action_persistence.feature
# Run only skill registry persistence scenarios
nox -s unit_tests -- features/skill_registry.feature
# Integration tests (Robot) -- runs all robot suites including persistence E2E
nox -s integration_tests
@@ -477,6 +531,9 @@ python -m robot robot/plan_persistence_e2e.robot
# Run only persistence lifecycle E2E
python -m robot robot/persistence_lifecycle.robot
# Run only skill registry smoke tests
nox -s integration_tests -- robot/skill_registry.robot
# Run only persistence robot alignment (Behave)
nox -s unit_tests -- features/persistence_robot_alignment.feature
```
@@ -594,6 +651,27 @@ Use `Resource ${CURDIR}/filename.resource` instead of bare `Resource filename.re
Ensure Robot tests use `${PYTHON}` (injected by nox from the venv) instead of bare `python` in `Run Process` calls.
### In-memory SQLite tests lose data between repository calls
When writing Behave tests that use `sqlite:///:memory:` with a `sessionmaker`,
do **not** create a new `Session` per repository call. SQLite in-memory databases
use `StaticPool` (a single shared connection). If each repository method creates
its own session, the abandoned sessions may be garbage-collected at unpredictable
times, triggering a `ROLLBACK` that discards previously flushed data.
**Fix:** Create a single shared `Session` at scenario setup and pass a factory
that always returns it:
```python
real_factory = sessionmaker(bind=engine)
shared_session = real_factory()
repo = MyRepository(session_factory=lambda: shared_session)
```
This mirrors the production `UnitOfWork` pattern (ADR-019) where all
repositories share one session within a transaction. See
`features/steps/skill_registry_steps.py` for a working example.
### Tests hang indefinitely
Add `timeout` parameters to `Run Process` calls in Robot tests:
+73
View File
@@ -89,3 +89,76 @@ svc.remove_skill("local/code-tools")
| item_config | Text | Optional JSON configuration |
| item_order | Integer | Stable ordering index |
| created_at | String(30) | ISO-8601 creation timestamp |
## Testing
### Behave BDD Tests
The Behave feature file `features/skill_registry.feature` contains 23
scenarios covering the full `SkillRegistryService` and `SkillRepository`
persistence layer:
| Area | Scenarios | Description |
|------|-----------|-------------|
| Registration | 6 | Each skill item type (`tool_ref`, `include`, `inline_tool`, `mcp_source`, `agent_source`) plus basic round-trip |
| Duplicate rejection | 1 | `DuplicateSkillError` on name collision |
| Retrieval | 2 | Get by name, non-existent returns `None` |
| Listing | 2 | List all, list with namespace filter |
| Update | 2 | Description change, non-existent raises `SkillNotFoundError` |
| Deletion | 2 | Remove existing (cascades items), remove non-existent returns `False` |
| Overrides | 1 | Override metadata survives persistence round-trip |
| Item ordering | 1 | Mixed item types maintain stable order across round-trip |
| Name validation | 3 | Empty name, no-slash name, special characters rejected at persistence layer |
| Large payload | 1 | 50 tool refs persist correctly |
| Domain objects | 1 | Listed skills are `Skill` domain instances |
Step definitions: `features/steps/skill_registry_steps.py`
**Session management note:** Each scenario creates a fresh in-memory
SQLite database with a single shared `Session` that is reused across all
repository calls within the scenario. This mirrors the production
`UnitOfWork` pattern (ADR-019) and avoids transaction-scoping issues
inherent to multiple ephemeral sessions on a single `StaticPool`
connection.
Run the Behave suite via:
```bash
nox -s unit_tests -- features/skill_registry.feature
```
### Robot Smoke Tests
A Robot Framework smoke suite validates core registry operations:
```
robot/skill_registry.robot
robot/helper_skill_registry.py
```
| Test Case | What it validates |
|---|---|
| Register And Retrieve A Skill | Round-trip: register then get by name |
| List Skills With Namespace Filter | Register multiple skills, filter by namespace |
| Update A Skill | Change description after registration |
| Reject Duplicate Skill Name | Duplicate name produces error |
| Remove A Skill | Remove and verify absence |
```bash
nox -s integration_tests -- robot/skill_registry.robot
```
### ASV Benchmarks
Performance benchmarks live in `benchmarks/skill_registry_bench.py`:
- **`SkillModelConstruction`** -- `time_from_domain` (domain-to-ORM mapping)
- **`SkillModelReconstruction`** -- `time_to_domain` (ORM-to-domain mapping)
- **`SkillRepositoryCRUD`** -- `time_create_skill`, `time_get_skill`,
`time_list_all`, `time_list_namespace`
- **`SkillRegistryListPerformance`** -- `time_list_100_skills`,
`time_list_100_skills_filtered`
```bash
nox -s benchmark
```
+149
View File
@@ -0,0 +1,149 @@
# Validation Pipeline — Apply Gating
## Overview
The validation apply gate enforces that attached validations pass before
the apply phase proceeds. Validations are attached to resources and run
automatically during the Execute-to-Apply transition. Required validation
failures block apply; informational failures are recorded but do not block.
## Attachment Model
### `ValidationAttachment`
Represents a validation bound to a resource with scope and mode.
| Field | Type | Description |
|-------------------|-------------------|--------------------------------------|
| `attachment_id` | `str` | Unique ULID for this attachment |
| `validation_name` | `str` | Name of the validation tool |
| `resource_id` | `str` | Target resource ID |
| `resource_name` | `str` | Human-readable resource name |
| `mode` | `ValidationMode` | `required` or `informational` |
| `scope` | `AttachmentScope` | `direct`, `project`, or `plan` |
| `scope_target` | `str` | Project name or plan ID for scoping |
| `arguments` | `dict[str, Any]` | Arguments passed to the validation |
### `AttachmentScope`
| Value | Description |
|-----------|----------------------------------------------------|
| `direct` | Always active for the resource |
| `project` | Active when resource is accessed via a project |
| `plan` | Active only for a specific plan |
When multiple scopes apply, the **union** of all applicable validations
is collected and run.
## Validation Modes
| Mode | Behavior on Failure |
|-----------------|--------------------------------------------------|
| `required` | Blocks apply, plan stays in Execute phase |
| `informational` | Recorded in summary but does **not** block apply |
## Result Models
### `ApplyValidationResult`
Per-validation execution result.
| Field | Type | Description |
|-------------------|------------------|-------------------------------------|
| `attachment_id` | `str` | Which attachment was evaluated |
| `validation_name` | `str` | Validation tool name |
| `resource_id` | `str` | Target resource |
| `mode` | `ValidationMode` | Required or informational |
| `passed` | `bool` | Whether the validation passed |
| `message` | `str` | Result message |
| `data` | `dict` | Additional structured data |
| `duration_ms` | `int` | Execution time in milliseconds |
| `error` | `str | None` | Error message if execution failed |
| `timed_out` | `bool` | Whether execution timed out |
### `ApplyValidationSummary`
Aggregated results with gating decision.
**Properties:** `total`, `required_passed`, `required_failed`,
`informational_passed`, `informational_failed`, `all_required_passed`,
`is_empty`
**Methods:**
- `to_plan_metadata()` — Dict for plan `validation_summary` field
- `format_cli_output()` — Human-readable summary for CLI display
## Runner Interface
### `ValidationRunner` (ABC)
Abstract interface for executing a single validation:
```python
def run_validation(
self,
attachment: ValidationAttachment,
context: dict[str, Any],
) -> ApplyValidationResult: ...
```
### `DefaultValidationRunner`
Default stub using text matching. Checks if the validation name or
argument values appear in the context. Production implementations
should invoke the actual validation tool via the tool execution pipeline.
## Apply Validation Gate
### `ApplyValidationGate`
Orchestrates the full validation-before-apply flow:
```python
gate = ApplyValidationGate(runner=DefaultValidationRunner())
summary = gate.run(plan_id, attachments, context)
if gate.should_block_apply(summary):
reasons = gate.get_failure_reasons(summary)
# Block apply, report reasons
else:
# Proceed to apply
```
### Example Output
When apply is **blocked**:
```
Validation Gate: BLOCKED
Required: 1 passed, 1 failed
Informational: 0 passed, 0 failed
Failures:
- lint-check on res-001: 3 lint errors found
```
When apply is **allowed**:
```
Validation Gate: PASSED
Required: 2 passed, 0 failed
Informational: 1 passed, 0 failed
```
## Integration with Plan Metadata
The validation summary is stored in the plan's `validation_summary` field
via `ApplyValidationSummary.to_plan_metadata()`:
```python
{
"total": 3,
"required_passed": 2,
"required_failed": 1,
"informational_passed": 0,
"informational_failed": 0,
"all_required_passed": false,
"evaluated_at": "2026-02-22T10:30:00+00:00",
"results": [...]
}
```
+24 -14
View File
@@ -6,7 +6,7 @@ integration layer.
from __future__ import annotations
from typing import Any
from typing import Any, Callable
from behave import given, then, when
from behave.runner import Context
@@ -36,24 +36,24 @@ from cleveragents.infrastructure.database.repositories import (
# ---------------------------------------------------------------------------
def _get_session_factory(context: Context) -> sessionmaker[Session]:
def _get_session_factory(context: Context) -> Callable[[], Session]:
"""Return the session factory stored on *context*.
The factory always returns the same shared session for the current
scenario, mirroring the ``UnitOfWork`` behaviour used in production.
"""
return context.skill_session_factory # type: ignore[no-any-return]
def _commit_pending(context: Context) -> None:
"""Commit pending transaction on the shared SQLite :memory: connection.
"""Commit the shared scenario session.
The repository creates its own session internally, so the flushed
data lives on the underlying connection's pending transaction. We
must bind a session to that connection and commit to make the data
durable across subsequent sessions.
Because the repository only *flushes* (never commits), the test
must explicitly commit to make data durable. The session is kept
open so that subsequent steps can continue using it.
"""
session = _get_session_factory(context)()
try:
session.execute(text("SELECT 1"))
session.commit()
finally:
session.close()
session.commit()
def _make_skill(
@@ -97,9 +97,19 @@ def step_clean_db(context: Context) -> None:
# SEC-3: Enable FK enforcement so CASCADE works at the DB level
event.listen(engine, "connect", _enable_fk_pragma)
Base.metadata.create_all(engine)
factory: sessionmaker[Session] = sessionmaker(bind=engine)
real_factory: sessionmaker[Session] = sessionmaker(bind=engine)
# Use a single shared session for the entire scenario.
# SkillRepository creates a session per method call via its factory;
# returning the *same* session mirrors the UnitOfWork behaviour used
# in production and prevents transaction-scoping issues that arise
# when multiple ephemeral sessions share one in-memory SQLite
# connection (StaticPool): abandoned sessions may be garbage-
# collected at unpredictable times, rolling back previously flushed
# but uncommitted data.
shared_session: Session = real_factory()
context.skill_engine = engine
context.skill_session_factory = factory
context.skill_session_factory = lambda: shared_session
context.skill_error = None
context.skill_result = None
context.skill_removal_result = None
+383
View File
@@ -0,0 +1,383 @@
"""Step definitions for validation apply gate tests."""
from __future__ import annotations
from behave import given, then, when
from behave.runner import Context
from cleveragents.application.services.validation_apply import (
ApplyValidationGate,
ApplyValidationResult,
ApplyValidationSummary,
AttachmentScope,
DefaultValidationRunner,
ValidationAttachment,
)
from cleveragents.domain.models.core.tool import ValidationMode
__all__: list[str] = []
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a validation apply test environment")
def step_given_val_apply_env(context: Context) -> None:
context.val_attachments = [] # list[ValidationAttachment]
context.val_run_context = {} # dict[str, str]
context.val_result = None # ApplyValidationResult | None
context.val_summary = None # ApplyValidationSummary | None
context.val_gate_summary = None # ApplyValidationSummary | None
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given('a validation attachment "{name}" for resource "{res}" with mode "{mode}"')
def step_given_attachment(context: Context, name: str, res: str, mode: str) -> None:
idx = len(context.val_attachments)
att = ValidationAttachment(
attachment_id=f"01ATT{idx:021d}",
validation_name=name,
resource_id=res,
mode=ValidationMode(mode),
)
context.val_attachments.append(att)
@given('a project-scoped attachment "{name}" for resource "{res}" targeting "{target}"')
def step_given_project_attachment(
context: Context, name: str, res: str, target: str
) -> None:
idx = len(context.val_attachments)
att = ValidationAttachment(
attachment_id=f"01ATT{idx:021d}",
validation_name=name,
resource_id=res,
mode=ValidationMode.REQUIRED,
scope=AttachmentScope.PROJECT,
scope_target=target,
)
context.val_attachments.append(att)
@given('a plan-scoped attachment "{name}" for resource "{res}" targeting "{target}"')
def step_given_plan_attachment(
context: Context, name: str, res: str, target: str
) -> None:
idx = len(context.val_attachments)
att = ValidationAttachment(
attachment_id=f"01ATT{idx:021d}",
validation_name=name,
resource_id=res,
mode=ValidationMode.REQUIRED,
scope=AttachmentScope.PLAN,
scope_target=target,
)
context.val_attachments.append(att)
@given('a validation run context with key "{key}" and value "{value}"')
def step_given_val_context(context: Context, key: str, value: str) -> None:
context.val_run_context[key] = value
@given('the attachment has argument "{key}" with value "{value}"')
def step_given_attachment_arg(context: Context, key: str, value: str) -> None:
context.val_attachments[-1].arguments[key] = value
@given("an apply validation summary with {p} required passed and {f} required failed")
def step_given_apply_summary(context: Context, p: str, f: str) -> None:
results: list[ApplyValidationResult] = []
for i in range(int(p)):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{i:021d}",
validation_name=f"pass-val-{i}",
resource_id=f"res-{i}",
mode=ValidationMode.REQUIRED,
passed=True,
message="ok",
)
)
for i in range(int(f)):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{int(p) + i:021d}",
validation_name=f"fail-val-{i}",
resource_id=f"res-{int(p) + i}",
mode=ValidationMode.REQUIRED,
passed=False,
message="failed check",
)
)
context.val_summary = ApplyValidationSummary(
plan_id="01VALTEST0000000000001",
results=results,
)
@given("the summary has {ip} informational passed and {iff} informational failed")
def step_given_informational_results(context: Context, ip: str, iff: str) -> None:
base = len(context.val_summary.results)
for i in range(int(ip)):
context.val_summary.results.append(
ApplyValidationResult(
attachment_id=f"01ATT{base + i:021d}",
validation_name=f"info-pass-{i}",
resource_id=f"res-info-{i}",
mode=ValidationMode.INFORMATIONAL,
passed=True,
message="info ok",
)
)
base2 = len(context.val_summary.results)
for i in range(int(iff)):
context.val_summary.results.append(
ApplyValidationResult(
attachment_id=f"01ATT{base2 + i:021d}",
validation_name=f"info-fail-{i}",
resource_id=f"res-info-fail-{i}",
mode=ValidationMode.INFORMATIONAL,
passed=False,
message="info failed",
)
)
@given("a validation run context that raises an error during iteration")
def step_given_error_context(context: Context) -> None:
"""Set up a context dict-like object whose items() raises."""
class _BadContext(dict): # type: ignore[type-arg]
def items(self) -> None: # type: ignore[override]
msg = "boom"
raise RuntimeError(msg)
context.val_run_context = _BadContext()
@given("an apply validation summary with a bare failure result")
def step_given_summary_bare_failure(context: Context) -> None:
context.val_summary = ApplyValidationSummary(
plan_id="01VALTEST0000000000001",
results=[
ApplyValidationResult(
attachment_id="01ATT000000000000000000000",
validation_name="bare-val",
resource_id="res-bare",
mode=ValidationMode.REQUIRED,
passed=False,
message="",
error=None,
),
],
)
context.val_gate_summary = context.val_summary
@given("an apply validation summary with an error result")
def step_given_summary_with_error(context: Context) -> None:
context.val_summary = ApplyValidationSummary(
plan_id="01VALTEST0000000000001",
results=[
ApplyValidationResult(
attachment_id="01ATT000000000000000000000",
validation_name="broken-val",
resource_id="res-broken",
mode=ValidationMode.REQUIRED,
passed=False,
message="",
error="execution timed out",
),
],
)
context.val_gate_summary = context.val_summary
@given("an empty apply validation summary")
def step_given_empty_apply_summary(context: Context) -> None:
context.val_summary = ApplyValidationSummary(
plan_id="01VALTEST0000000000001",
results=[],
)
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when("I execute the validation with DefaultValidationRunner")
def step_when_execute_default_runner(context: Context) -> None:
runner = DefaultValidationRunner()
context.val_result = runner.run_validation(
context.val_attachments[-1],
context.val_run_context,
)
@when('I run the apply validation gate for plan "{plan_id}"')
def step_when_run_gate(context: Context, plan_id: str) -> None:
runner = DefaultValidationRunner()
gate = ApplyValidationGate(runner=runner)
context.val_gate_summary = gate.run(
plan_id=plan_id,
attachments=context.val_attachments,
context=context.val_run_context,
)
# Also store as val_summary for summary assertions
context.val_summary = context.val_gate_summary
@when('I run the apply validation gate for plan "{plan_id}" with no attachments')
def step_when_run_gate_empty(context: Context, plan_id: str) -> None:
runner = DefaultValidationRunner()
gate = ApplyValidationGate(runner=runner)
context.val_gate_summary = gate.run(
plan_id=plan_id,
attachments=[],
context=context.val_run_context,
)
context.val_summary = context.val_gate_summary
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then('the attachment mode should be "{mode}"')
def step_then_attachment_mode(context: Context, mode: str) -> None:
att = context.val_attachments[-1]
assert att.mode == ValidationMode(mode), (
f"Expected mode '{mode}', got '{att.mode.value}'"
)
@then('the attachment scope should be "{scope}"')
def step_then_attachment_scope(context: Context, scope: str) -> None:
att = context.val_attachments[-1]
assert att.scope == AttachmentScope(scope), (
f"Expected scope '{scope}', got '{att.scope.value}'"
)
@then('the attachment scope target should be "{target}"')
def step_then_scope_target(context: Context, target: str) -> None:
att = context.val_attachments[-1]
assert att.scope_target == target
@then("the validation result should be passed")
def step_then_val_result_passed(context: Context) -> None:
assert context.val_result is not None
assert context.val_result.passed, (
f"Expected passed, got failed: {context.val_result.message}"
)
@then("the validation result should be failed")
def step_then_val_result_failed(context: Context) -> None:
assert context.val_result is not None
assert not context.val_result.passed
@then("the validation result duration_ms should be >= {n}")
def step_then_val_result_duration(context: Context, n: str) -> None:
assert context.val_result is not None
assert context.val_result.duration_ms >= int(n), (
f"Expected duration_ms >= {n}, got {context.val_result.duration_ms}"
)
@then("the validation result should have an error message")
def step_then_val_result_has_error(context: Context) -> None:
assert context.val_result is not None
assert context.val_result.error is not None and context.val_result.error != "", (
f"Expected error message, got: {context.val_result.error!r}"
)
@then("the apply summary should report all required passed")
def step_then_all_required_passed(context: Context) -> None:
assert context.val_summary is not None
assert context.val_summary.all_required_passed
@then("the apply summary should not report all required passed")
def step_then_not_all_required_passed(context: Context) -> None:
assert context.val_summary is not None
assert not context.val_summary.all_required_passed
@then("the apply summary total should be {n}")
def step_then_apply_summary_total(context: Context, n: str) -> None:
assert context.val_summary is not None
assert context.val_summary.total == int(n), (
f"Expected total {n}, got {context.val_summary.total}"
)
@then("the apply summary required failed count should be {n}")
def step_then_required_failed(context: Context, n: str) -> None:
assert context.val_summary is not None
assert context.val_summary.required_failed == int(n)
@then("the apply summary informational failed count should be {n}")
def step_then_informational_failed(context: Context, n: str) -> None:
assert context.val_summary is not None
assert context.val_summary.informational_failed == int(n)
@then("the apply summary should be empty")
def step_then_apply_summary_empty(context: Context) -> None:
assert context.val_summary is not None
assert context.val_summary.is_empty
@then('the plan metadata should have key "{key}"')
def step_then_plan_metadata_key(context: Context, key: str) -> None:
assert context.val_summary is not None
md = context.val_summary.to_plan_metadata()
assert key in md, f"Key '{key}' not in {list(md.keys())}"
@then('the validation gate CLI output should contain "{text}"')
def step_then_val_gate_cli_output(context: Context, text: str) -> None:
assert context.val_summary is not None
output = context.val_summary.format_cli_output()
assert text in output, f"Expected '{text}' in:\n{output}"
@then("the gate should allow apply")
def step_then_gate_allows(context: Context) -> None:
assert context.val_gate_summary is not None
gate = ApplyValidationGate(runner=DefaultValidationRunner())
assert not gate.should_block_apply(context.val_gate_summary), (
"Expected gate to allow apply but it blocked"
)
@then("the gate should block apply")
def step_then_gate_blocks(context: Context) -> None:
assert context.val_gate_summary is not None
gate = ApplyValidationGate(runner=DefaultValidationRunner())
assert gate.should_block_apply(context.val_gate_summary), (
"Expected gate to block apply but it allowed"
)
@then('the gate failure reasons should mention "{text}"')
def step_then_failure_reasons(context: Context, text: str) -> None:
assert context.val_gate_summary is not None
gate = ApplyValidationGate(runner=DefaultValidationRunner())
reasons = gate.get_failure_reasons(context.val_gate_summary)
combined = " ".join(reasons)
assert text in combined, f"Expected '{text}' in failure reasons: {reasons}"
+145
View File
@@ -0,0 +1,145 @@
Feature: Validation apply gate
As a plan executor
I want attached validations to run before apply
So that required validation failures block apply
Background:
Given a validation apply test environment
# -- Attachment model ----------------------------------------------------
Scenario: Create a required validation attachment
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
Then the attachment mode should be "required"
And the attachment scope should be "direct"
Scenario: Create an informational validation attachment
Given a validation attachment "style-check" for resource "res-002" with mode "informational"
Then the attachment mode should be "informational"
Scenario: Create a project-scoped attachment
Given a project-scoped attachment "lint-check" for resource "res-001" targeting "my-project"
Then the attachment scope should be "project"
And the attachment scope target should be "my-project"
Scenario: Create a plan-scoped attachment
Given a plan-scoped attachment "lint-check" for resource "res-001" targeting "01PLAN0001"
Then the attachment scope should be "plan"
And the attachment scope target should be "01PLAN0001"
# -- DefaultValidationRunner ---------------------------------------------
Scenario: DefaultValidationRunner passes when context matches
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation run context with key "lint-check" and value "passed"
When I execute the validation with DefaultValidationRunner
Then the validation result should be passed
Scenario: DefaultValidationRunner fails when context has no match
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation run context with key "status" and value "no matches"
When I execute the validation with DefaultValidationRunner
Then the validation result should be failed
Scenario: DefaultValidationRunner matches via arguments
Given a validation attachment "coverage-check" for resource "res-001" with mode "required"
And the attachment has argument "threshold" with value "90"
And a validation run context with key "coverage" and value "90 percent"
When I execute the validation with DefaultValidationRunner
Then the validation result should be passed
# -- ApplyValidationSummary ----------------------------------------------
Scenario: Summary all required passed with no failures
Given an apply validation summary with 2 required passed and 0 required failed
Then the apply summary should report all required passed
And the apply summary total should be 2
Scenario: Summary blocked when required failure exists
Given an apply validation summary with 1 required passed and 1 required failed
Then the apply summary should not report all required passed
And the apply summary required failed count should be 1
Scenario: Summary with informational results does not block
Given an apply validation summary with 0 required passed and 0 required failed
And the summary has 1 informational passed and 1 informational failed
Then the apply summary should report all required passed
And the apply summary informational failed count should be 1
Scenario: Empty summary reports all required passed
Given an empty apply validation summary
Then the apply summary should be empty
And the apply summary should report all required passed
Scenario: Summary to_plan_metadata has expected fields
Given an apply validation summary with 1 required passed and 1 required failed
Then the plan metadata should have key "total"
And the plan metadata should have key "all_required_passed"
And the plan metadata should have key "results"
Scenario: Summary format_cli_output shows BLOCKED on failures
Given an apply validation summary with 0 required passed and 1 required failed
Then the validation gate CLI output should contain "BLOCKED"
Scenario: Summary format_cli_output shows no details when neither message nor error
Given an apply validation summary with a bare failure result
Then the validation gate CLI output should contain "no details"
Scenario: Summary format_cli_output shows error in failure detail
Given an apply validation summary with an error result
Then the validation gate CLI output should contain "execution timed out"
Scenario: Summary format_cli_output shows PASSED when all pass
Given an apply validation summary with 2 required passed and 0 required failed
Then the validation gate CLI output should contain "PASSED"
Scenario: DefaultValidationRunner populates duration_ms
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation run context with key "lint-check" and value "passed"
When I execute the validation with DefaultValidationRunner
Then the validation result duration_ms should be >= 0
Scenario: DefaultValidationRunner handles exception during execution
Given a validation attachment "err-val" for resource "res-err" with mode "required"
And a validation run context that raises an error during iteration
When I execute the validation with DefaultValidationRunner
Then the validation result should be failed
And the validation result should have an error message
# -- ApplyValidationGate -------------------------------------------------
Scenario: Gate allows apply when all required pass
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation run context with key "lint-check" and value "ok"
When I run the apply validation gate for plan "01PLAN0001"
Then the gate should allow apply
Scenario: Gate blocks apply when required validation fails
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation run context with key "other" and value "irrelevant"
When I run the apply validation gate for plan "01PLAN0001"
Then the gate should block apply
And the gate failure reasons should mention "lint-check"
Scenario: Gate allows apply despite informational failures
Given a validation attachment "style-check" for resource "res-001" with mode "informational"
And a validation run context with key "other" and value "irrelevant"
When I run the apply validation gate for plan "01PLAN0001"
Then the gate should allow apply
Scenario: Gate with multiple attachments mixed results
Given a validation attachment "lint-check" for resource "res-001" with mode "required"
And a validation attachment "style-check" for resource "res-001" with mode "informational"
And a validation run context with key "lint-check" and value "passed"
When I run the apply validation gate for plan "01PLAN0001"
Then the gate should allow apply
And the apply summary total should be 2
Scenario: Gate failure reasons include error text when result has error
Given an apply validation summary with an error result
Then the gate failure reasons should mention "error:"
Scenario: Gate with no attachments allows apply
When I run the apply validation gate for plan "01PLAN0001" with no attachments
Then the gate should allow apply
And the apply summary should be empty
+16 -16
View File
@@ -4163,22 +4163,22 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled
**PARALLEL SUBTRACK D1.cli [Jeff]**: Validation summary output in plan status
**SEQUENTIAL MERGE NOTE**: D1.apply lands before D1.cli.
- [ ] **COMMIT (Owner: Luis | Group: D1.apply | Branch: feature/m3-validation-apply | Planned: Day 15 | Expected: Day 21) - Commit message: "feat(apply): run validation attachments"**
- [ ] Git [Luis]: `git checkout master`
- [ ] Git [Luis]: `git pull origin master`
- [ ] Git [Luis]: `git checkout -b feature/m3-validation-apply`
- [ ] Git [Luis]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
- [ ] Code [Luis]: Add `ValidationRunner` integration that executes attached validations before Apply.
- [ ] Code [Luis]: Block Apply on `required` validation failures and record results in plan metadata.
- [ ] Docs [Luis]: Add `docs/reference/validation_pipeline.md` with required vs informational behavior.
- [ ] Tests (Behave) [Luis]: Add scenarios for required failure blocking and informational pass-through.
- [ ] Tests (Robot) [Luis]: Add Robot test that attaches a validation and verifies apply gating.
- [ ] Tests (ASV) [Luis]: Add `benchmarks/validation_pipeline_bench.py` for validation runtime baseline.
- [ ] Quality [Luis]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
- [ ] Quality [Luis]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes.
- [ ] Git [Luis]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index
- [ ] Git [Luis]: `git commit -m "feat(apply): run validation attachments"`
- [ ] Git [Luis]: `git push -u origin feature/m3-validation-apply`
- [X] **COMMIT (Owner: Luis | Group: D1.apply | Branch: feature/m3-validation-apply | Planned: Day 15 | Expected: Day 21) - Commit message: "feat(apply): run validation attachments"**
- [X] Git [Luis]: `git checkout master`
- [X] Git [Luis]: `git pull origin master`
- [X] Git [Luis]: `git checkout -b feature/m3-validation-apply`
- [X] Git [Luis]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
- [X] Code [Luis]: Add `ValidationRunner` integration that executes attached validations before Apply.
- [X] Code [Luis]: Block Apply on `required` validation failures and record results in plan metadata.
- [X] Docs [Luis]: Add `docs/reference/validation_pipeline.md` with required vs informational behavior.
- [X] Tests (Behave) [Luis]: Add scenarios for required failure blocking and informational pass-through.
- [X] Tests (Robot) [Luis]: Add Robot test that attaches a validation and verifies apply gating.
- [X] Tests (ASV) [Luis]: Add `benchmarks/validation_pipeline_bench.py` for validation runtime baseline.
- [X] Quality [Luis]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
- [X] Quality [Luis]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes.
- [X] Git [Luis]: Perform an appropriate `git add` command to add all the files that should be part of this commit to the git index
- [X] Git [Luis]: `git commit -m "feat(apply): run validation attachments"`
- [X] Git [Luis]: `git push -u origin feature/m3-validation-apply`
- [ ] Forgejo PR [Luis]: Open PR from `feature/m3-validation-apply` to `master` with a suitable and thorough description
- [ ] **COMMIT (Owner: Jeff | Group: D1.cli | Branch: feature/m3-validation-status | Planned: Day 16 | Expected: Day 22) - Commit message: "feat(cli): surface validation summaries"**
+163
View File
@@ -0,0 +1,163 @@
"""Robot Framework helper for validation apply gate.
Provides a CLI-style interface for Robot to invoke validation
attachment creation, runner execution, and gate evaluation.
Exit code 0 = success, 1 = failure.
Usage::
python robot/helper_validation_apply.py run-gate \\
<plan_id> <val_name> <res_id> <mode> <ctx_key> <ctx_val>
python robot/helper_validation_apply.py summary \\
<plan_id> <n_req_pass> <n_req_fail>
python robot/helper_validation_apply.py check-block \\
<plan_id> <n_req_pass> <n_req_fail>
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
# Ensure the src directory is on the import path.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.application.services.validation_apply import ( # noqa: E402
ApplyValidationGate,
ApplyValidationResult,
ApplyValidationSummary,
DefaultValidationRunner,
ValidationAttachment,
)
from cleveragents.domain.models.core.tool import ValidationMode # noqa: E402
def _cmd_run_gate(args: list[str]) -> int:
"""Run the validation gate with a single attachment."""
if len(args) < 6:
print(
"Usage: run-gate <plan_id> <val_name> <res_id> <mode> <ctx_key> <ctx_val>"
)
return 1
plan_id = args[0]
att = ValidationAttachment(
attachment_id="01ATT000000000000000000000",
validation_name=args[1],
resource_id=args[2],
mode=ValidationMode(args[3]),
)
ctx = {args[4]: args[5]}
runner = DefaultValidationRunner()
gate = ApplyValidationGate(runner=runner)
summary = gate.run(plan_id, [att], ctx)
blocked = gate.should_block_apply(summary)
status = "BLOCKED" if blocked else "ALLOWED"
print(f"val-gate-ok: {status}")
print(f" total={summary.total}")
print(f" required_passed={summary.required_passed}")
print(f" required_failed={summary.required_failed}")
return 0
def _cmd_summary(args: list[str]) -> int:
"""Build a summary and print metadata."""
if len(args) < 3:
print("Usage: summary <plan_id> <n_req_pass> <n_req_fail>")
return 1
plan_id = args[0]
n_pass, n_fail = int(args[1]), int(args[2])
results: list[ApplyValidationResult] = []
for i in range(n_pass):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{i:021d}",
validation_name=f"val-{i}",
resource_id=f"res-{i}",
mode=ValidationMode.REQUIRED,
passed=True,
message="ok",
)
)
for i in range(n_fail):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{n_pass + i:021d}",
validation_name=f"val-{n_pass + i}",
resource_id=f"res-{n_pass + i}",
mode=ValidationMode.REQUIRED,
passed=False,
message="failed",
)
)
summary = ApplyValidationSummary(plan_id=plan_id, results=results)
md = summary.to_plan_metadata()
print(f"val-summary-ok: {json.dumps(md, default=str)}")
return 0
def _cmd_check_block(args: list[str]) -> int:
"""Check if gate blocks apply."""
if len(args) < 3:
print("Usage: check-block <plan_id> <n_req_pass> <n_req_fail>")
return 1
plan_id = args[0]
n_pass, n_fail = int(args[1]), int(args[2])
results: list[ApplyValidationResult] = []
for i in range(n_pass):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{i:021d}",
validation_name=f"val-{i}",
resource_id=f"res-{i}",
mode=ValidationMode.REQUIRED,
passed=True,
message="ok",
)
)
for i in range(n_fail):
results.append(
ApplyValidationResult(
attachment_id=f"01ATT{n_pass + i:021d}",
validation_name=f"val-{n_pass + i}",
resource_id=f"res-{n_pass + i}",
mode=ValidationMode.REQUIRED,
passed=False,
message="failed",
)
)
summary = ApplyValidationSummary(plan_id=plan_id, results=results)
gate = ApplyValidationGate(runner=DefaultValidationRunner())
blocked = gate.should_block_apply(summary)
status = "BLOCKED" if blocked else "ALLOWED"
print(f"val-check-block-ok: {status}")
return 0
def main() -> int:
"""Entry point called by Robot Framework ``Run Process``."""
if len(sys.argv) < 2:
print("Usage: helper_validation_apply.py <run-gate|summary|check-block> ...")
return 1
command = sys.argv[1]
args = sys.argv[2:]
dispatch = {
"run-gate": _cmd_run_gate,
"summary": _cmd_summary,
"check-block": _cmd_check_block,
}
handler = dispatch.get(command)
if handler is None:
print(f"Unknown command: {command}")
return 1
return handler(args)
if __name__ == "__main__":
sys.exit(main())
+49
View File
@@ -0,0 +1,49 @@
*** Settings ***
Documentation Integration tests for validation apply gate
Resource common.resource
Library Process
Library OperatingSystem
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Test Cases ***
Gate Allows Apply When Validation Passes
[Documentation] Verify gate allows apply when required validation passes
${result}= Run Process ${PYTHON} ${CURDIR}/helper_validation_apply.py
... run-gate 01PLAN0001 lint-check res-001 required lint-check passed
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} val-gate-ok: ALLOWED
Gate Blocks Apply When Validation Fails
[Documentation] Verify gate blocks apply when required validation fails
${result}= Run Process ${PYTHON} ${CURDIR}/helper_validation_apply.py
... run-gate 01PLAN0001 lint-check res-001 required other irrelevant
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} val-gate-ok: BLOCKED
Build Validation Summary Metadata
[Documentation] Verify validation summary produces plan metadata
${result}= Run Process ${PYTHON} ${CURDIR}/helper_validation_apply.py
... summary 01PLAN0001 2 1
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} val-summary-ok:
Should Contain ${result.stdout} all_required_passed
Check Block With No Failures
[Documentation] Verify gate reports ALLOWED with no failures
${result}= Run Process ${PYTHON} ${CURDIR}/helper_validation_apply.py
... check-block 01PLAN0001 3 0
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} val-check-block-ok: ALLOWED
Check Block With Failures
[Documentation] Verify gate reports BLOCKED with failures
${result}= Run Process ${PYTHON} ${CURDIR}/helper_validation_apply.py
... check-block 01PLAN0001 1 2
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} val-check-block-ok: BLOCKED
@@ -12,9 +12,25 @@ from cleveragents.application.services.skill_registry_service import (
from cleveragents.application.services.tool_registry_service import (
ToolRegistryService,
)
from cleveragents.application.services.validation_apply import (
ApplyValidationGate,
ApplyValidationResult,
ApplyValidationSummary,
AttachmentScope,
DefaultValidationRunner,
ValidationAttachment,
ValidationRunner,
)
__all__ = [
"ApplyValidationGate",
"ApplyValidationResult",
"ApplyValidationSummary",
"AttachmentScope",
"DefaultValidationRunner",
"PersistentSessionService",
"SkillRegistryService",
"ToolRegistryService",
"ValidationAttachment",
"ValidationRunner",
]
@@ -0,0 +1,406 @@
"""Validation apply gate: run attached validations before apply.
Orchestrates the execution of attached validations before the apply phase
and gates apply based on required/informational validation outcomes.
Core types:
- **ValidationAttachment**: A reference to a validation attached to a resource.
- **ApplyValidationResult**: Per-validation pass/fail result.
- **ApplyValidationSummary**: Aggregated results with gating decision.
- **ValidationRunner**: Protocol for executing a single validation.
- **DefaultValidationRunner**: Stub implementation using text matching.
- **ApplyValidationGate**: Orchestrates validation execution and gating.
Usage::
gate = ApplyValidationGate(runner=DefaultValidationRunner())
summary = gate.run(attachments, plan_context)
if summary.all_required_passed:
# proceed to apply
else:
# block apply, record failures
"""
from __future__ import annotations
import time
from abc import ABC, abstractmethod
from datetime import UTC, datetime
from enum import StrEnum
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from cleveragents.domain.models.core.tool import ValidationMode
# ---------------------------------------------------------------------------
# Enums
# ---------------------------------------------------------------------------
class AttachmentScope(StrEnum):
"""Scope of a validation attachment."""
DIRECT = "direct"
PROJECT = "project"
PLAN = "plan"
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
class ValidationAttachment(BaseModel):
"""A validation attached to a resource for apply gating.
Represents a single validation-to-resource binding with scope
and mode information.
"""
attachment_id: str = Field(..., description="Unique attachment ULID")
validation_name: str = Field(..., min_length=1, description="Validation tool name")
resource_id: str = Field(..., description="Target resource ID")
resource_name: str = Field(default="", description="Target resource name")
mode: ValidationMode = Field(
default=ValidationMode.REQUIRED,
description="Whether failure blocks apply",
)
scope: AttachmentScope = Field(
default=AttachmentScope.DIRECT,
description="Attachment scope level",
)
scope_target: str = Field(
default="",
description="Scope target (project name or plan ID)",
)
arguments: dict[str, Any] = Field(
default_factory=dict,
description="Arguments to pass to the validation",
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class ApplyValidationResult(BaseModel):
"""Result of executing a single validation."""
attachment_id: str = Field(..., description="Attachment that was evaluated")
validation_name: str = Field(..., description="Validation tool name")
resource_id: str = Field(..., description="Resource the validation ran on")
mode: ValidationMode = Field(..., description="Required or informational")
passed: bool = Field(..., description="Whether the validation passed")
message: str = Field(default="", description="Result message")
data: dict[str, Any] = Field(
default_factory=dict, description="Additional result data"
)
duration_ms: int = Field(default=0, ge=0, description="Execution time in ms")
error: str | None = Field(
default=None, description="Error message if execution failed"
)
timed_out: bool = Field(default=False, description="Whether execution timed out")
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class ApplyValidationSummary(BaseModel):
"""Aggregated results from all validations run before apply."""
plan_id: str = Field(..., description="Plan ULID")
results: list[ApplyValidationResult] = Field(
default_factory=list,
description="Per-validation results",
)
evaluated_at: datetime = Field(
default_factory=lambda: datetime.now(tz=UTC),
description="When the validation gate was evaluated",
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
@property
def total(self) -> int:
"""Total number of validations run."""
return len(self.results)
@property
def required_passed(self) -> int:
"""Number of required validations that passed."""
return sum(
1 for r in self.results if r.mode == ValidationMode.REQUIRED and r.passed
)
@property
def required_failed(self) -> int:
"""Number of required validations that failed."""
return sum(
1
for r in self.results
if r.mode == ValidationMode.REQUIRED and not r.passed
)
@property
def informational_passed(self) -> int:
"""Number of informational validations that passed."""
return sum(
1
for r in self.results
if r.mode == ValidationMode.INFORMATIONAL and r.passed
)
@property
def informational_failed(self) -> int:
"""Number of informational validations that failed."""
return sum(
1
for r in self.results
if r.mode == ValidationMode.INFORMATIONAL and not r.passed
)
@property
def all_required_passed(self) -> bool:
"""Whether all required validations passed."""
return self.required_failed == 0
@property
def is_empty(self) -> bool:
"""Whether no validations were run."""
return self.total == 0
def to_plan_metadata(self) -> dict[str, Any]:
"""Convert to a dict for plan validation_summary field.
Compatible with the plan model's ``validation_summary`` field.
"""
return {
"total": self.total,
"required_passed": self.required_passed,
"required_failed": self.required_failed,
"informational_passed": self.informational_passed,
"informational_failed": self.informational_failed,
"all_required_passed": self.all_required_passed,
"evaluated_at": self.evaluated_at.isoformat(),
"results": [
{
"attachment_id": r.attachment_id,
"validation_name": r.validation_name,
"resource_id": r.resource_id,
"mode": r.mode.value,
"passed": r.passed,
"message": r.message,
"duration_ms": r.duration_ms,
"error": r.error,
"timed_out": r.timed_out,
}
for r in self.results
],
}
def format_cli_output(self) -> str:
"""Format a human-readable summary for CLI display."""
lines: list[str] = []
status = "PASSED" if self.all_required_passed else "BLOCKED"
lines.append(f"Validation Gate: {status}")
lines.append(
f" Required: {self.required_passed} passed, {self.required_failed} failed"
)
lines.append(
f" Informational: {self.informational_passed} passed, "
f"{self.informational_failed} failed"
)
if self.required_failed > 0:
lines.append(" Failures:")
for r in self.results:
if r.mode == ValidationMode.REQUIRED and not r.passed:
msg = r.message or r.error or "no details"
lines.append(f" - {r.validation_name} on {r.resource_id}: {msg}")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Runner interface
# ---------------------------------------------------------------------------
class ValidationRunner(ABC):
"""Interface for executing a single validation.
Implementations handle the actual invocation of a validation tool
(e.g., subprocess, MCP call, in-process function).
"""
@abstractmethod
def run_validation(
self,
attachment: ValidationAttachment,
context: dict[str, Any],
) -> ApplyValidationResult:
"""Execute a single validation.
Args:
attachment: The validation attachment to execute.
context: Plan context (changeset summary, metadata, etc.).
Returns:
The validation result.
"""
...
class DefaultValidationRunner(ValidationRunner):
"""Default validation runner using text matching.
For each attachment, checks if the validation name appears in the
context values. This is a simple default for development and testing.
Production implementations should use the full tool execution pipeline.
"""
def run_validation(
self,
attachment: ValidationAttachment,
context: dict[str, Any],
) -> ApplyValidationResult:
"""Execute a validation via text matching against context.
Checks if the validation name (or any argument value) appears
in the context. This is a placeholder real implementations
invoke the actual validation tool.
"""
start = time.monotonic()
try:
context_text = " ".join(f"{k}={v}" for k, v in context.items()).lower()
val_name = attachment.validation_name.lower()
# Simple heuristic: check if validation name is referenced in context
passed = val_name in context_text
if not passed:
# Check arguments
for val in attachment.arguments.values():
if str(val).lower() in context_text:
passed = True
break
elapsed = int((time.monotonic() - start) * 1000)
return ApplyValidationResult(
attachment_id=attachment.attachment_id,
validation_name=attachment.validation_name,
resource_id=attachment.resource_id,
mode=attachment.mode,
passed=passed,
message="matched" if passed else "not matched",
duration_ms=elapsed,
)
except Exception as exc:
elapsed = int((time.monotonic() - start) * 1000)
return ApplyValidationResult(
attachment_id=attachment.attachment_id,
validation_name=attachment.validation_name,
resource_id=attachment.resource_id,
mode=attachment.mode,
passed=False,
message="",
error=str(exc),
duration_ms=elapsed,
)
# ---------------------------------------------------------------------------
# Apply validation gate
# ---------------------------------------------------------------------------
class ApplyValidationGate:
"""Orchestrates validation execution and apply gating.
Collects applicable validation attachments, runs them via the
provided runner, and produces an ``ApplyValidationSummary``
that determines whether apply should proceed.
"""
def __init__(self, runner: ValidationRunner) -> None:
"""Initialise the gate with a validation runner.
Args:
runner: The ``ValidationRunner`` to execute each validation.
"""
self._runner = runner
def run(
self,
plan_id: str,
attachments: list[ValidationAttachment],
context: dict[str, Any],
) -> ApplyValidationSummary:
"""Run all attached validations and return gating result.
Executes each validation attachment against the provided context
and returns an aggregated summary.
Args:
plan_id: The plan ULID.
attachments: List of validation attachments to evaluate.
context: Plan context for evaluation.
Returns:
An ``ApplyValidationSummary`` with per-validation results
and the overall gating decision.
"""
results: list[ApplyValidationResult] = []
for attachment in attachments:
result = self._runner.run_validation(attachment, context)
results.append(result)
return ApplyValidationSummary(
plan_id=plan_id,
results=results,
)
def should_block_apply(
self,
summary: ApplyValidationSummary,
) -> bool:
"""Determine if apply should be blocked.
Args:
summary: The validation summary to check.
Returns:
``True`` if apply should be blocked (required failures exist).
"""
return not summary.all_required_passed
def get_failure_reasons(
self,
summary: ApplyValidationSummary,
) -> list[str]:
"""Extract human-readable failure reasons from a summary.
Args:
summary: The validation summary.
Returns:
List of failure reason strings for required failures.
"""
reasons: list[str] = []
for r in summary.results:
if r.mode == ValidationMode.REQUIRED and not r.passed:
if r.error:
reasons.append(
f"{r.validation_name} on {r.resource_id}: error: {r.error}"
)
else:
reasons.append(
f"{r.validation_name} on {r.resource_id}: "
f"{r.message or 'failed'}"
)
return reasons