cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

feat(dod): enforce definition-of-done gating

Browse Source 
ISSUES CLOSED: #178
This commit is contained in:
CoreRasurae
2026-02-22 11:32:59 +00:00
parent 4874f2ad6f
commit fe69d132f1
9 changed files with 1507 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/dod_evaluation_bench.py
+119
View File
@@ -0,0 +1,119 @@
"""ASV benchmarks for definition-of-done evaluation throughput.
Measures the performance of:
- DoD text parsing into criteria
- Template rendering with argument substitution
- TextMatchEvaluator evaluation
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
# Ensure the local *source* tree is importable.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.domain.models.core.definition_of_done import ( # noqa: E402
TextMatchEvaluator,
parse_dod_criteria,
render_dod_template,
)
# ---------------------------------------------------------------------------
# Benchmark: parse_dod_criteria
# ---------------------------------------------------------------------------
_SIMPLE_DOD = "All tests pass\nNo lint errors\nCoverage above 90%"
_BULLET_DOD = "- All tests pass\n- No lint errors\n- Coverage above 90%\n- No warnings"
_NUMBERED_DOD = (
"1. Tests pass\n2. Lint clean\n3. Coverage >= 97%\n4. Types check\n5. Docs updated"
)
class ParseDoDCriteria:
"""Benchmark suite for ``parse_dod_criteria``."""
def time_parse_simple(self) -> None:
"""Parse plain-text DoD criteria."""
parse_dod_criteria(_SIMPLE_DOD)
def time_parse_bullets(self) -> None:
"""Parse bullet-point DoD criteria."""
parse_dod_criteria(_BULLET_DOD)
def time_parse_numbered(self) -> None:
"""Parse numbered DoD criteria."""
parse_dod_criteria(_NUMBERED_DOD)
def time_parse_empty(self) -> None:
"""Parse empty DoD text."""
parse_dod_criteria("")
# ---------------------------------------------------------------------------
# Benchmark: render_dod_template
# ---------------------------------------------------------------------------
_TEMPLATE = "Tests for {module} in {env} must pass with {threshold}% coverage"
_ARGS = {"module": "auth", "env": "production", "threshold": "95"}
class RenderDoDTemplate:
"""Benchmark suite for ``render_dod_template``."""
def time_render_with_args(self) -> None:
"""Render template with all placeholders matched."""
render_dod_template(_TEMPLATE, _ARGS)
def time_render_partial_args(self) -> None:
"""Render template with only some placeholders matched."""
render_dod_template(_TEMPLATE, {"module": "auth"})
def time_render_no_args(self) -> None:
"""Render template with empty arguments dict."""
render_dod_template(_TEMPLATE, {})
# ---------------------------------------------------------------------------
# Benchmark: TextMatchEvaluator
# ---------------------------------------------------------------------------
class TextMatchEvaluation:
"""Benchmark suite for ``TextMatchEvaluator.evaluate``."""
def setup(self) -> None:
"""Prepare evaluator and criteria."""
self.evaluator = TextMatchEvaluator()
self.criteria = parse_dod_criteria(_NUMBERED_DOD)
self.context_all_pass = {
"tests": "tests pass",
"lint": "lint clean",
"coverage": "coverage >= 97%",
"types": "types check",
"docs": "docs updated",
}
self.context_mixed = {
"tests": "tests pass",
"lint": "3 lint errors",
}
def time_evaluate_all_pass(self) -> None:
"""Evaluate criteria where all pass."""
self.evaluator.evaluate(self.criteria, self.context_all_pass)
def time_evaluate_mixed_results(self) -> None:
"""Evaluate criteria with mixed pass/fail."""
self.evaluator.evaluate(self.criteria, self.context_mixed)
def time_evaluate_empty_context(self) -> None:
"""Evaluate criteria with empty context (all skip)."""
self.evaluator.evaluate(self.criteria, {})
docs/reference/definition_of_done.md
+115
View File
@@ -0,0 +1,115 @@
# Definition-of-Done Gating
## Overview
The definition-of-done (DoD) subsystem enforces completion criteria before
a plan is permitted to apply changes. Each plan may carry a
`definition_of_done` text field listing testable criteria that must be
satisfied before the apply phase proceeds.
## Domain Models
### `DoDStatus`
A `StrEnum` with three values:
| Value | Meaning |
|-----------|------------------------------------------|
| `passed` | Criterion was met |
| `failed` | Criterion was **not** met |
| `skipped` | Criterion could not be evaluated |
### `DoDCriterion`
A single testable criterion parsed from DoD text.
| Field | Type | Description |
|----------|--------------|-------------------------------------------|
| `index` | `int` | Zero-based position in the DoD list |
| `text` | `str` | Human-readable criterion text |
| `pattern`| `str | None` | Optional regex pattern for evaluation |
### `DoDResult`
Evaluation result for a single criterion.
| Field | Type | Description |
|-------------|----------------|--------------------------------------|
| `criterion` | `DoDCriterion` | The evaluated criterion |
| `status` | `DoDStatus` | Pass/fail/skip result |
| `reasoning` | `str` | Explanation for the evaluation |
### `DoDSummary`
Aggregated evaluation results for all criteria in a plan.
| Field | Type | Description |
|---------------------|-------------------|---------------------------------|
| `plan_id` | `str` | Plan ULID |
| `definition_of_done`| `str` | Original DoD text |
| `results` | `list[DoDResult]` | Per-criterion results |
| `evaluated_at` | `datetime` | Evaluation timestamp |
**Properties:** `total`, `passed`, `failed`, `skipped`, `all_passed`, `is_empty`
**Methods:**
- `to_cli_dict()` — Dictionary for CLI display
- `to_validation_summary()` — Dict compatible with plan `validation_summary`
## Functions
### `parse_dod_criteria(definition_of_done: str) -> list[DoDCriterion]`
Parses DoD text into individual criteria. Supports:
- Plain text (one criterion per line)
- Bullet points (`-` or `*` prefix)
- Numbered lists (`1.`, `2.` prefix)
- Empty/whitespace-only input returns empty list
### `render_dod_template(template: str, arguments: dict) -> str`
Substitutes `{placeholder}` markers in a DoD template with values from
the arguments dict. Unmatched placeholders are preserved as-is.
## Evaluator Interface
### `DoDEvaluator` (ABC)
Abstract base class for DoD evaluation. Implementations must provide:
```python
def evaluate(
self,
criteria: list[DoDCriterion],
context: dict[str, Any],
) -> list[DoDResult]: ...
```
### `TextMatchEvaluator`
Default evaluator using text/regex matching:
- If a criterion has a `pattern`, uses regex matching against context values
- Otherwise, checks if criterion text appears in any context key or value
- Empty context causes all criteria to be marked `SKIPPED`
- Invalid regex patterns result in `FAILED` status with error reasoning
## Integration with Plan Lifecycle
The DoD evaluation result feeds into the plan's `validation_summary` via
`DoDSummary.to_validation_summary()`, which produces a dict compatible
with the existing validation gate format:
```python
{
"total": 3,
"required_passed": 2,
"required_failed": 1,
"informational_passed": 0,
"informational_failed": 0,
"dod_evaluated": True,
"dod_all_passed": False,
}
```
When `dod_all_passed` is `False`, the apply phase is blocked.
features/definition_of_done.feature
+185
View File
@@ -0,0 +1,185 @@
Feature: Definition-of-Done gating
As a plan executor
I want apply to be blocked unless all DoD criteria are met
So that only plans meeting completion criteria are committed
Background:
Given a definition of done test environment
# -- DoD parsing ---------------------------------------------------------
Scenario: Parse DoD text into criteria
When I parse the DoD text "All tests pass\nNo lint errors\nCoverage above 90%"
Then there should be 3 criteria parsed
And criterion 0 text should be "All tests pass"
And criterion 1 text should be "No lint errors"
Scenario: Parse DoD with bullet points
When I parse the DoD text "- Tests pass\n- No errors\n* Coverage ok"
Then there should be 3 criteria parsed
And criterion 0 text should be "Tests pass"
Scenario: Parse DoD with numbered items
When I parse the DoD text "1. Tests pass\n2. No errors"
Then there should be 2 criteria parsed
And criterion 0 text should be "Tests pass"
And criterion 1 text should be "No errors"
Scenario: Parse empty DoD returns no criteria
When I parse an empty DoD text
Then there should be 0 criteria parsed
# -- Template rendering --------------------------------------------------
Scenario: Render DoD template with plan arguments
When I render the DoD template "Tests for {module} must pass" with argument "module" as "auth"
Then the rendered DoD should be "Tests for auth must pass"
Scenario: Render DoD template preserves unmatched placeholders
When I render the DoD template "Deploy {service} to {env}" with argument "service" as "api"
Then the rendered DoD should contain "{env}"
# -- Text match evaluation -----------------------------------------------
Scenario: TextMatchEvaluator passes when context contains criterion text
Given DoD criteria from "All tests pass"
And evaluation context with key "status" value "all tests pass"
When I evaluate the criteria with TextMatchEvaluator
Then all criteria should have passed
Scenario: TextMatchEvaluator fails when context lacks criterion text
Given DoD criteria from "All tests pass"
And evaluation context with key "status" value "3 tests failed"
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "failed"
Scenario: TextMatchEvaluator with regex pattern passes on match
Given a DoD criterion with text "Coverage check" and pattern "coverage.*9[0-9]"
And evaluation context with key "coverage" value "coverage=95%"
When I evaluate the criteria with TextMatchEvaluator
Then all criteria should have passed
Scenario: TextMatchEvaluator with regex pattern fails on no match
Given a DoD criterion with text "Coverage check" and pattern "coverage.*9[0-9]"
And evaluation context with key "coverage" value "coverage=80%"
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "failed"
Scenario: TextMatchEvaluator skips when context is empty
Given DoD criteria from "All tests pass"
And an empty evaluation context
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "skipped"
Scenario: TextMatchEvaluator handles invalid regex gracefully
Given a DoD criterion with text "Bad regex" and pattern "[invalid"
And evaluation context with key "data" value "anything"
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "failed"
And criterion 0 reasoning should contain "Invalid regex"
# -- DoDSummary model ----------------------------------------------------
Scenario: DoDSummary reports all_passed when no failures
Given a DoDSummary with 2 passed and 0 failed
Then the summary should report all passed
And the DoD summary total should be 2
Scenario: DoDSummary reports not all passed when failures exist
Given a DoDSummary with 1 passed and 1 failed
Then the summary should not report all passed
And the summary failed count should be 1
Scenario: DoDSummary empty check
Given an empty DoDSummary
Then the summary should be empty
And the summary should not report all passed
Scenario: DoDSummary to_cli_dict has expected fields
Given a DoDSummary with 1 passed and 0 failed
Then the summary CLI dict should have key "total"
And the summary CLI dict should have key "all_passed"
And the summary CLI dict should have key "criteria"
Scenario: DoDSummary to_validation_summary is compatible
Given a DoDSummary with 2 passed and 1 failed
Then the validation summary should have required_passed 2
And the validation summary should have required_failed 1
And the validation summary should have dod_evaluated true
# -- Evaluation context matching -----------------------------------------
Scenario: TextMatchEvaluator matches criterion text in context keys
Given DoD criteria from "lint_errors"
And evaluation context with key "lint_errors" value "0"
When I evaluate the criteria with TextMatchEvaluator
Then all criteria should have passed
Scenario: Multiple criteria with mixed results
Given DoD criteria from "tests pass\nlint errors"
And evaluation context with key "result" value "tests pass, no issues"
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "passed"
And criterion 1 should have status "failed"
# -- Coverage: edge cases ------------------------------------------------
Scenario: Parse DoD with blank lines between criteria
When I parse the DoD text "Tests pass\n\nNo errors"
Then there should be 2 criteria parsed
Scenario: Parse DoD with whitespace only line
When I parse the DoD text " \n- Tests pass"
Then there should be 1 criteria parsed
And criterion 0 text should be "Tests pass"
Scenario: TextMatchEvaluator matches via individual key lookup
Given DoD criteria from "all_tests_passed"
And evaluation context with key "all_tests_passed" value "true"
When I evaluate the criteria with TextMatchEvaluator
Then all criteria should have passed
# -- T1: all-SKIPPED summary gate behavior --------------------------------
Scenario: DoDSummary all_passed is false when all criteria skipped
Given a DoDSummary with all criteria skipped
Then the summary should not report all passed
And the summary skipped count should be 2
# -- T2: bullet prefix yielding empty text --------------------------------
Scenario: Parse DoD with bullet prefix but no text
When I parse the DoD text "- \n- Tests pass"
Then there should be 1 criteria parsed
And criterion 0 text should be "Tests pass"
# -- T5: multi-argument template rendering --------------------------------
Scenario: Render DoD template with multiple arguments
When I render the DoD template "{svc} in {env} OK" with arguments "svc"="api" and "env"="prod"
Then the rendered DoD should be "api in prod OK"
Scenario: Render DoD template prevents injection via values
When I render the DoD template "Test {name}" with argument "name" as "{secret}"
Then the rendered DoD should be "Test {secret}"
# -- S3: length limit on template -----------------------------------------
Scenario: Render DoD rejects oversized template
Given a DoD template that is 11000 characters long
When I render the oversized DoD template with argument "x" as "y"
Then the DoD render should fail with "exceeds maximum"
Scenario: Render DoD rejects oversized output
Given a short DoD template with a huge argument value
When I render the DoD template that produces oversized output
Then the DoD render should fail with "exceeds maximum"
# -- S2: ReDoS protection -------------------------------------------------
Scenario: TextMatchEvaluator rejects oversized regex pattern
Given a DoD criterion with text "long regex" and pattern that is 600 chars
And evaluation context with key "data" value "anything"
When I evaluate the criteria with TextMatchEvaluator
Then criterion 0 should have status "failed"
And criterion 0 reasoning should contain "exceeds maximum"
features/steps/definition_of_done_steps.py
+327
View File
@@ -0,0 +1,327 @@
"""Step definitions for definition-of-done gating tests."""
from __future__ import annotations
from behave import given, then, when
from behave.runner import Context
from cleveragents.domain.models.core.definition_of_done import (
DoDCriterion,
DoDResult,
DoDStatus,
DoDSummary,
TextMatchEvaluator,
parse_dod_criteria,
render_dod_template,
)
__all__: list[str] = []
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a definition of done test environment")
def step_given_dod_env(context: Context) -> None:
context.dod_criteria = None # list[DoDCriterion] | None
context.dod_rendered = None # str | None
context.dod_results = None # list[DoDResult] | None
context.dod_context = {} # dict[str, Any]
context.dod_summary = None # DoDSummary | None
# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------
@given('DoD criteria from "{text}"')
def step_given_criteria_from(context: Context, text: str) -> None:
resolved = text.replace("\\n", "\n")
context.dod_criteria = parse_dod_criteria(resolved)
@given('evaluation context with key "{key}" value "{value}"')
def step_given_eval_context(context: Context, key: str, value: str) -> None:
context.dod_context[key] = value
@given("an empty evaluation context")
def step_given_empty_context(context: Context) -> None:
context.dod_context = {}
@given('a DoD criterion with text "{text}" and pattern "{pattern}"')
def step_given_criterion_with_pattern(
context: Context, text: str, pattern: str
) -> None:
context.dod_criteria = [DoDCriterion(index=0, text=text, pattern=pattern)]
@given("a DoDSummary with {p} passed and {f} failed")
def step_given_summary(context: Context, p: str, f: str) -> None:
results: list[DoDResult] = []
for i in range(int(p)):
results.append(
DoDResult(
criterion=DoDCriterion(index=i, text=f"crit-{i}"),
status=DoDStatus.PASSED,
reasoning="ok",
)
)
for i in range(int(f)):
results.append(
DoDResult(
criterion=DoDCriterion(index=int(p) + i, text=f"fail-{i}"),
status=DoDStatus.FAILED,
reasoning="not ok",
)
)
context.dod_summary = DoDSummary(
plan_id="01DODTEST0000000000001",
definition_of_done="test dod",
results=results,
)
@given("an empty DoDSummary")
def step_given_empty_summary(context: Context) -> None:
context.dod_summary = DoDSummary(
plan_id="01DODTEST0000000000001",
definition_of_done="empty",
results=[],
)
@given("a DoDSummary with all criteria skipped")
def step_given_all_skipped_summary(context: Context) -> None:
results = [
DoDResult(
criterion=DoDCriterion(index=i, text=f"skip-{i}"),
status=DoDStatus.SKIPPED,
reasoning="skipped",
)
for i in range(2)
]
context.dod_summary = DoDSummary(
plan_id="01DODTEST0000000000001",
definition_of_done="skip test",
results=results,
)
@given("a DoD template that is {n} characters long")
def step_given_long_dod_template(context: Context, n: str) -> None:
context.dod_oversized_template = "x" * int(n)
@given('a DoD criterion with text "{text}" and pattern that is {n} chars')
def step_given_long_pattern(context: Context, text: str, n: str) -> None:
context.dod_criteria = [
DoDCriterion(index=0, text=text, pattern="a" * int(n)),
]
@given("a short DoD template with a huge argument value")
def step_given_huge_value_template(context: Context) -> None:
context.dod_short_template = "Result: {val}"
context.dod_huge_value = "x" * 60_000
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when('I parse the DoD text "{text}"')
def step_when_parse(context: Context, text: str) -> None:
resolved = text.replace("\\n", "\n")
context.dod_criteria = parse_dod_criteria(resolved)
@when("I parse an empty DoD text")
def step_when_parse_empty(context: Context) -> None:
context.dod_criteria = parse_dod_criteria("")
@when('I render the DoD template "{template}" with argument "{key}" as "{value}"')
def step_when_render(context: Context, template: str, key: str, value: str) -> None:
context.dod_rendered = render_dod_template(template, {key: value})
@when(
'I render the DoD template "{template}" with arguments "{k1}"="{v1}" and "{k2}"="{v2}"'
)
def step_when_render_multi(
context: Context, template: str, k1: str, v1: str, k2: str, v2: str
) -> None:
context.dod_rendered = render_dod_template(template, {k1: v1, k2: v2})
@when('I render the oversized DoD template with argument "{key}" as "{value}"')
def step_when_render_oversized(context: Context, key: str, value: str) -> None:
try:
context.dod_rendered = render_dod_template(
context.dod_oversized_template, {key: value}
)
context.dod_render_error = None
except ValueError as exc:
context.dod_render_error = exc
@when("I render the DoD template that produces oversized output")
def step_when_render_oversized_output(context: Context) -> None:
try:
context.dod_rendered = render_dod_template(
context.dod_short_template, {"val": context.dod_huge_value}
)
context.dod_render_error = None
except ValueError as exc:
context.dod_render_error = exc
@when("I evaluate the criteria with TextMatchEvaluator")
def step_when_evaluate(context: Context) -> None:
evaluator = TextMatchEvaluator()
context.dod_results = evaluator.evaluate(context.dod_criteria, context.dod_context)
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then("there should be {n} criteria parsed")
def step_then_criteria_count(context: Context, n: str) -> None:
assert context.dod_criteria is not None
assert len(context.dod_criteria) == int(n), (
f"Expected {n} criteria, got {len(context.dod_criteria)}"
)
@then('criterion {i} text should be "{text}"')
def step_then_criterion_text(context: Context, i: str, text: str) -> None:
assert context.dod_criteria is not None
idx = int(i)
assert idx < len(context.dod_criteria)
assert context.dod_criteria[idx].text == text, (
f"Expected text '{text}', got '{context.dod_criteria[idx].text}'"
)
@then('the rendered DoD should be "{expected}"')
def step_then_rendered(context: Context, expected: str) -> None:
assert context.dod_rendered == expected, (
f"Expected '{expected}', got '{context.dod_rendered}'"
)
@then('the rendered DoD should contain "{text}"')
def step_then_rendered_contains(context: Context, text: str) -> None:
assert context.dod_rendered is not None
assert text in context.dod_rendered
@then("all criteria should have passed")
def step_then_all_passed(context: Context) -> None:
assert context.dod_results is not None
for r in context.dod_results:
assert r.status == DoDStatus.PASSED, (
f"Criterion '{r.criterion.text}' has status '{r.status.value}'"
)
@then('criterion {i} should have status "{status}"')
def step_then_criterion_status(context: Context, i: str, status: str) -> None:
assert context.dod_results is not None
idx = int(i)
assert idx < len(context.dod_results)
assert context.dod_results[idx].status == DoDStatus(status), (
f"Expected status '{status}', got '{context.dod_results[idx].status.value}'"
)
@then('criterion {i} reasoning should contain "{text}"')
def step_then_criterion_reasoning(context: Context, i: str, text: str) -> None:
assert context.dod_results is not None
idx = int(i)
assert text in context.dod_results[idx].reasoning, (
f"Expected reasoning to contain '{text}', "
f"got: {context.dod_results[idx].reasoning}"
)
@then("the summary should report all passed")
def step_then_summary_all_passed(context: Context) -> None:
assert context.dod_summary is not None
assert context.dod_summary.all_passed
@then("the summary should not report all passed")
def step_then_summary_not_all_passed(context: Context) -> None:
assert context.dod_summary is not None
assert not context.dod_summary.all_passed
@then("the DoD summary total should be {n}")
def step_then_dod_summary_total(context: Context, n: str) -> None:
assert context.dod_summary is not None
assert context.dod_summary.total == int(n)
@then("the summary failed count should be {n}")
def step_then_summary_failed(context: Context, n: str) -> None:
assert context.dod_summary is not None
assert context.dod_summary.failed == int(n)
@then("the summary should be empty")
def step_then_summary_empty(context: Context) -> None:
assert context.dod_summary is not None
assert context.dod_summary.is_empty
@then('the summary CLI dict should have key "{key}"')
def step_then_cli_dict_key(context: Context, key: str) -> None:
assert context.dod_summary is not None
cli_dict = context.dod_summary.to_cli_dict()
assert key in cli_dict
@then("the validation summary should have required_passed {n}")
def step_then_val_summary_passed(context: Context, n: str) -> None:
assert context.dod_summary is not None
vs = context.dod_summary.to_validation_summary()
assert vs["required_passed"] == int(n)
@then("the validation summary should have required_failed {n}")
def step_then_val_summary_failed(context: Context, n: str) -> None:
assert context.dod_summary is not None
vs = context.dod_summary.to_validation_summary()
assert vs["required_failed"] == int(n)
@then("the validation summary should have dod_evaluated true")
def step_then_val_summary_dod(context: Context) -> None:
assert context.dod_summary is not None
vs = context.dod_summary.to_validation_summary()
assert vs["dod_evaluated"] is True
@then("the summary skipped count should be {n}")
def step_then_summary_skipped(context: Context, n: str) -> None:
assert context.dod_summary is not None
assert context.dod_summary.skipped == int(n), (
f"Expected skipped={n}, got {context.dod_summary.skipped}"
)
@then('the DoD render should fail with "{text}"')
def step_then_dod_render_error(context: Context, text: str) -> None:
assert context.dod_render_error is not None, "Expected a ValueError"
assert text in str(context.dod_render_error), (
f"Expected '{text}' in error: {context.dod_render_error}"
)
robot/definition_of_done.robot
+100
View File
@@ -0,0 +1,100 @@
*** Settings ***
Documentation Integration tests for definition-of-done gating
Resource common.resource
Library Process
Library OperatingSystem
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
*** Test Cases ***
Parse DoD Text Into Criteria
[Documentation] Verify DoD text is parsed into individual criteria
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... parse All tests pass\\nNo lint errors\\nCoverage above 90%
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} dod-parse-ok: 3 criteria
Should Contain ${result.stdout} All tests pass
Render DoD Template With Arguments
[Documentation] Verify DoD template rendering substitutes placeholders
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... render Tests for {module} must pass module auth
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} dod-render-ok: Tests for auth must pass
Evaluate DoD Criteria Against Context
[Documentation] Verify TextMatchEvaluator evaluates criteria correctly
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... evaluate All tests pass status all tests pass
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} dod-evaluate-ok: 1 passed, 0 failed
Build DoDSummary CLI Dict
[Documentation] Verify DoDSummary produces correct CLI dict output
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... summary 01PLANTEST0000000000001 test dod 2 1
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} dod-summary-ok:
Should Contain ${result.stdout} all_passed
Build DoDSummary Validation Summary
[Documentation] Verify DoDSummary produces compatible validation summary
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... validate-summary 01PLANTEST0000000000001 test dod 3 0
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} dod-validate-summary-ok:
Should Contain ${result.stdout} dod_evaluated
Helper Shows Usage On No Command
[Documentation] Verify helper prints usage when no command given
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
Helper Rejects Unknown Command
[Documentation] Verify helper rejects an unknown sub-command
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... bogus stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Unknown command
Parse Shows Usage On No Args
[Documentation] Verify parse sub-command prints usage with no arguments
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... parse stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
Render Shows Usage On Insufficient Args
[Documentation] Verify render sub-command prints usage with insufficient args
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... render template_only stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
Evaluate Shows Usage On Insufficient Args
[Documentation] Verify evaluate sub-command prints usage with insufficient args
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... evaluate dod_only stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
Summary Shows Usage On Insufficient Args
[Documentation] Verify summary sub-command prints usage with insufficient args
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... summary plan_id_only stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
Validate Summary Shows Usage On Insufficient Args
[Documentation] Verify validate-summary sub-command prints usage with insufficient args
${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py
... validate-summary plan_id_only stderr=STDOUT
Should Be Equal As Integers ${result.rc} 1
Should Contain ${result.stdout} Usage
robot/helper_definition_of_done.py
+175
View File
@@ -0,0 +1,175 @@
"""Robot Framework helper for definition-of-done evaluation.
Provides a CLI-style interface for Robot to invoke DoD parsing,
template rendering, and evaluation. Exit code 0 = success, 1 = failure.
Usage:
python robot/helper_definition_of_done.py parse <dod_text>
python robot/helper_definition_of_done.py render <template> <key> <value>
python robot/helper_definition_of_done.py evaluate <dod_text> <ctx_key> <ctx_value>
python robot/helper_definition_of_done.py summary <plan_id> <dod> <pass> <fail>
python robot/helper_definition_of_done.py validate-summary <plan_id> <dod> <p> <f>
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
# Ensure the src directory is on the import path.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.domain.models.core.definition_of_done import ( # noqa: E402
DoDCriterion,
DoDResult,
DoDStatus,
DoDSummary,
TextMatchEvaluator,
parse_dod_criteria,
render_dod_template,
)
def _cmd_parse(args: list[str]) -> int:
"""Parse DoD text and print criteria count."""
if len(args) < 1:
print("Usage: parse <dod_text>")
return 1
dod_text = args[0].replace("\\n", "\n")
criteria = parse_dod_criteria(dod_text)
print(f"dod-parse-ok: {len(criteria)} criteria")
for c in criteria:
print(f" [{c.index}] {c.text}")
return 0
def _cmd_render(args: list[str]) -> int:
"""Render a DoD template and print result."""
if len(args) < 3:
print("Usage: render <template> <key> <value>")
return 1
template, key, value = args[0], args[1], args[2]
rendered = render_dod_template(template, {key: value})
print(f"dod-render-ok: {rendered}")
return 0
def _cmd_evaluate(args: list[str]) -> int:
"""Evaluate DoD criteria against a context."""
if len(args) < 3:
print("Usage: evaluate <dod_text> <ctx_key> <ctx_value>")
return 1
dod_text = args[0].replace("\\n", "\n")
ctx_key, ctx_value = args[1], args[2]
criteria = parse_dod_criteria(dod_text)
evaluator = TextMatchEvaluator()
results = evaluator.evaluate(criteria, {ctx_key: ctx_value})
passed = sum(1 for r in results if r.status == DoDStatus.PASSED)
failed = sum(1 for r in results if r.status == DoDStatus.FAILED)
print(f"dod-evaluate-ok: {passed} passed, {failed} failed")
for r in results:
print(f" [{r.criterion.index}] {r.status.value}: {r.criterion.text}")
return 0
def _cmd_summary(args: list[str]) -> int:
"""Build a DoDSummary and print CLI dict."""
if len(args) < 4:
print("Usage: summary <plan_id> <dod_text> <n_passed> <n_failed>")
return 1
plan_id, dod_text = args[0], args[1]
n_passed, n_failed = int(args[2]), int(args[3])
results: list[DoDResult] = []
for i in range(n_passed):
results.append(
DoDResult(
criterion=DoDCriterion(index=i, text=f"crit-{i}"),
status=DoDStatus.PASSED,
reasoning="ok",
)
)
for i in range(n_failed):
results.append(
DoDResult(
criterion=DoDCriterion(index=n_passed + i, text=f"fail-{i}"),
status=DoDStatus.FAILED,
reasoning="not ok",
)
)
summary = DoDSummary(
plan_id=plan_id,
definition_of_done=dod_text,
results=results,
)
cli_dict = summary.to_cli_dict()
print(f"dod-summary-ok: {json.dumps(cli_dict, default=str)}")
return 0
def _cmd_validate_summary(args: list[str]) -> int:
"""Build a DoDSummary and print validation summary."""
if len(args) < 4:
print("Usage: validate-summary <plan_id> <dod_text> <n_passed> <n_failed>")
return 1
plan_id, dod_text = args[0], args[1]
n_passed, n_failed = int(args[2]), int(args[3])
results: list[DoDResult] = []
for i in range(n_passed):
results.append(
DoDResult(
criterion=DoDCriterion(index=i, text=f"crit-{i}"),
status=DoDStatus.PASSED,
reasoning="ok",
)
)
for i in range(n_failed):
results.append(
DoDResult(
criterion=DoDCriterion(index=n_passed + i, text=f"fail-{i}"),
status=DoDStatus.FAILED,
reasoning="not ok",
)
)
summary = DoDSummary(
plan_id=plan_id,
definition_of_done=dod_text,
results=results,
)
vs = summary.to_validation_summary()
print(f"dod-validate-summary-ok: {json.dumps(vs)}")
return 0
def main() -> int:
"""Entry point called by Robot Framework ``Run Process``."""
if len(sys.argv) < 2:
print(
"Usage: helper_definition_of_done.py "
"<parse|render|evaluate|summary|validate-summary> ..."
)
return 1
command = sys.argv[1]
args = sys.argv[2:]
dispatch = {
"parse": _cmd_parse,
"render": _cmd_render,
"evaluate": _cmd_evaluate,
"summary": _cmd_summary,
"validate-summary": _cmd_validate_summary,
}
handler = dispatch.get(command)
if handler is None:
print(f"Unknown command: {command}")
return 1
return handler(args)
if __name__ == "__main__":
sys.exit(main())
src/cleveragents/cli/commands/plan.py
+27
View File
@@ -139,6 +139,12 @@ def _plan_spec_dict(plan: Any) -> dict[str, object]:
{"text": inv.text, "source": inv.source.value}
for inv in plan.invariants
]
if plan.validation_summary and plan.validation_summary.get("dod_evaluated"):
result["dod_evaluation"] = {
"all_passed": plan.validation_summary.get("dod_all_passed", False),
"required_passed": plan.validation_summary.get("required_passed", 0),
"required_failed": plan.validation_summary.get("required_failed", 0),
}
if plan.error_message:
result["error_message"] = plan.error_message
return result
@@ -1145,6 +1151,27 @@ def _print_lifecycle_plan(plan: Any, title: str = "Plan") -> None:
f"[bold]Execution Actor:[/bold] {plan.execution_actor or '(not set)'}\n"
)
# Definition-of-Done evaluation summary
if plan.definition_of_done:
dod_preview = plan.definition_of_done[:200]
if len(plan.definition_of_done) > 200:
dod_preview += "..."
details += f"[bold]Definition of Done:[/bold]\n {dod_preview}\n"
if plan.validation_summary and plan.validation_summary.get("dod_evaluated"):
vs = plan.validation_summary
dod_passed = vs.get("dod_all_passed", False)
req_pass = vs.get("required_passed", 0)
req_fail = vs.get("required_failed", 0)
dod_total = req_pass + req_fail
status_color = "green" if dod_passed else "red"
status_label = "PASSED" if dod_passed else "FAILED"
details += (
f"[bold]DoD Evaluation:[/bold] "
f"[{status_color}]{status_label}[/{status_color}]"
f" ({req_pass}/{dod_total} passed"
f"{f', {req_fail} failed' if req_fail else ''})\n"
)
# Optional actors
if plan.estimation_actor:
details += f"[bold]Estimation Actor:[/bold] {plan.estimation_actor}\n"
src/cleveragents/domain/models/core/__init__.py
+20
View File
@@ -49,6 +49,18 @@ from cleveragents.domain.models.core.correction import (
CorrectionStatus,
)
from cleveragents.domain.models.core.debug_attempt import DebugAttempt
# Definition-of-Done models
from cleveragents.domain.models.core.definition_of_done import (
DoDCriterion,
DoDEvaluator,
DoDResult,
DoDStatus,
DoDSummary,
TextMatchEvaluator,
parse_dod_criteria,
render_dod_template,
)
from cleveragents.domain.models.core.diff_review import (
DiffBuilder,
DiffEntry,
@@ -209,6 +221,11 @@ __all__ = [
"DiffBuilder",
"DiffEntry",
"DiffSerializer",
"DoDCriterion",
"DoDEvaluator",
"DoDResult",
"DoDStatus",
"DoDSummary",
"GuardResult",
"InMemoryChangeSetStore",
"InMemoryInvocationTracker",
@@ -279,6 +296,7 @@ __all__ = [
"SpecChangeSet",
"SummaryForUpdateContextParams",
"TemporalScope",
"TextMatchEvaluator",
"Tool",
"ToolCapability",
"ToolInvocation",
@@ -292,5 +310,7 @@ __all__ = [
"get_builtin_profile",
"merge_invariants",
"normalize_change_path",
"parse_dod_criteria",
"parse_namespaced_name",
"render_dod_template",
]
src/cleveragents/domain/models/core/definition_of_done.py
+439
View File
@@ -0,0 +1,439 @@
"""Definition-of-Done evaluation domain models.
Provides structured evaluation of plan completion criteria
(``definition_of_done``) before apply is permitted.
Core types:
- **DoDCriterion**: A single testable criterion parsed from the DoD text.
- **DoDResult**: Pass/fail result for a single criterion with reasoning.
- **DoDSummary**: Aggregated evaluation results for all criteria.
- **DoDEvaluator**: Interface for evaluating DoD criteria.
- **TextMatchEvaluator**: Default evaluator using regex/substring matching.
- **render_dod_template**: Renders a DoD template with plan arguments.
"""
from __future__ import annotations
import logging
import re
from abc import ABC, abstractmethod
from datetime import datetime, timezone
from enum import StrEnum
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
logger = logging.getLogger(__name__)
#: Maximum length for a DoD template string (prevent DoS).
MAX_DOD_TEMPLATE_LENGTH: int = 10_000
#: Maximum length for a rendered DoD output string (prevent DoS).
MAX_DOD_OUTPUT_LENGTH: int = 50_000
#: Timeout (in effective characters) for regex evaluation to
#: limit catastrophic backtracking. Patterns exceeding this
#: length are rejected.
MAX_REGEX_PATTERN_LENGTH: int = 500
#: Safe placeholder regex: ``{simple_name}`` only.
_DOD_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")
# ---------------------------------------------------------------------------
# Enums
# ---------------------------------------------------------------------------
class DoDStatus(StrEnum):
"""Evaluation status for a DoD criterion."""
PASSED = "passed"
FAILED = "failed"
SKIPPED = "skipped"
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
class DoDCriterion(BaseModel):
"""A single testable criterion parsed from definition_of_done text.
Each line or bullet point in the DoD becomes a separate criterion.
"""
index: int = Field(..., ge=0, description="Zero-based criterion index")
text: str = Field(..., min_length=1, description="Criterion text")
pattern: str | None = Field(
default=None,
description=(
"Optional regex pattern for evaluation. "
"If None, substring matching is used."
),
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class DoDResult(BaseModel):
"""Pass/fail result for a single DoD criterion."""
criterion: DoDCriterion = Field(..., description="The criterion that was evaluated")
status: DoDStatus = Field(..., description="Whether the criterion passed or failed")
reasoning: str = Field(
default="", description="Explanation for the evaluation result"
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
class DoDSummary(BaseModel):
"""Aggregated evaluation results for all DoD criteria."""
plan_id: str = Field(..., description="Plan ULID")
definition_of_done: str = Field(..., description="The original DoD text")
results: list[DoDResult] = Field(
default_factory=list,
description="Per-criterion evaluation results",
)
evaluated_at: datetime = Field(
default_factory=lambda: datetime.now(tz=timezone.utc), # noqa: UP017
description="When the evaluation was performed (UTC)",
)
model_config = ConfigDict(
str_strip_whitespace=True,
validate_assignment=True,
)
def _counts(self) -> tuple[int, int, int]:
"""Return (passed, failed, skipped) counts in a single pass."""
p = f = s = 0
for r in self.results:
if r.status == DoDStatus.PASSED:
p += 1
elif r.status == DoDStatus.FAILED:
f += 1
else:
s += 1
return p, f, s
@property
def total(self) -> int:
"""Total number of criteria evaluated."""
return len(self.results)
@property
def passed(self) -> int:
"""Number of criteria that passed."""
return self._counts()[0]
@property
def failed(self) -> int:
"""Number of criteria that failed."""
return self._counts()[1]
@property
def skipped(self) -> int:
"""Number of criteria that were skipped."""
return self._counts()[2]
@property
def all_passed(self) -> bool:
"""Whether every criterion passed.
Returns ``False`` when any criterion failed, was skipped, or
when the results list is empty. Only returns ``True`` when
*every* criterion has status ``PASSED``.
"""
p, f, s = self._counts()
return p > 0 and f == 0 and s == 0
@property
def is_empty(self) -> bool:
"""Whether no criteria were evaluated."""
return self.total == 0
def to_cli_dict(self) -> dict[str, Any]:
"""Convert to dictionary for CLI display."""
return {
"plan_id": self.plan_id,
"total": self.total,
"passed": self.passed,
"failed": self.failed,
"skipped": self.skipped,
"all_passed": self.all_passed,
"evaluated_at": self.evaluated_at.isoformat(),
"criteria": [
{
"index": r.criterion.index,
"text": r.criterion.text,
"status": r.status.value,
"reasoning": r.reasoning,
}
for r in self.results
],
}
def to_validation_summary(self) -> dict[str, Any]:
"""Convert to a validation_summary dict for plan metadata.
Compatible with the plan's ``validation_summary`` field format.
"""
return {
"total": self.total,
"required_passed": self.passed,
"required_failed": self.failed,
"informational_passed": 0,
"informational_failed": 0,
"dod_evaluated": True,
"dod_all_passed": self.all_passed,
}
# ---------------------------------------------------------------------------
# Template rendering
# ---------------------------------------------------------------------------
def render_dod_template(
template: str,
arguments: dict[str, Any],
) -> str:
"""Render a DoD template with plan arguments.
Substitutes ``{arg_name}`` placeholders in the template with
values from the arguments dict using **single-pass** regex
substitution (preventing injection through argument values).
Unmatched placeholders are preserved as-is.
Args:
template: The DoD template text with ``{placeholder}`` markers.
arguments: Plan argument values to substitute.
Returns:
The rendered DoD text.
Raises:
ValueError: If the template or rendered output exceeds size
limits.
"""
if len(template) > MAX_DOD_TEMPLATE_LENGTH:
raise ValueError(
f"DoD template length {len(template)} exceeds maximum "
f"{MAX_DOD_TEMPLATE_LENGTH}"
)
def _replace(match: re.Match[str]) -> str:
key = match.group(1)
if key in arguments:
return str(arguments[key])
return match.group(0)
result = _DOD_PLACEHOLDER_RE.sub(_replace, template)
if len(result) > MAX_DOD_OUTPUT_LENGTH:
raise ValueError(
f"Rendered DoD length {len(result)} exceeds maximum {MAX_DOD_OUTPUT_LENGTH}"
)
return result
def parse_dod_criteria(
definition_of_done: str,
) -> list[DoDCriterion]:
"""Parse a definition_of_done text into individual criteria.
Each non-empty line (after stripping leading bullet markers like
``-``, ``*``, ``1.``) becomes a separate criterion.
Args:
definition_of_done: The DoD text to parse.
Returns:
List of ``DoDCriterion`` objects.
"""
if not definition_of_done or not definition_of_done.strip():
return []
criteria: list[DoDCriterion] = []
for line in definition_of_done.strip().splitlines():
# Strip leading whitespace and bullet markers
text = line.strip()
if not text:
continue
# Remove common bullet prefixes: "- ", "* ", "1. ", "2. ", etc.
text = re.sub(r"^[-*]\s*", "", text)
text = re.sub(r"^\d+\.\s*", "", text)
text = text.strip()
if not text:
continue
criteria.append(DoDCriterion(index=len(criteria), text=text))
return criteria
# ---------------------------------------------------------------------------
# Evaluator interface
# ---------------------------------------------------------------------------
class DoDEvaluator(ABC):
"""Abstract interface for evaluating DoD criteria.
Implementations check whether each criterion in a DoD is met,
returning a ``DoDResult`` for each.
"""
@abstractmethod
def evaluate(
self,
criteria: list[DoDCriterion],
context: dict[str, Any],
) -> list[DoDResult]:
"""Evaluate a list of criteria against a context.
Args:
criteria: Parsed DoD criteria.
context: Evaluation context (e.g., changeset summary,
validation results, file list).
Returns:
A ``DoDResult`` for each criterion.
"""
...
class TextMatchEvaluator(DoDEvaluator):
"""Default DoD evaluator using text matching.
For each criterion, checks whether the criterion text (or its
regex pattern) matches any value in the context dict. This is
a simple but effective default for DoD checks like:
- "All tests pass" → checks if context has "tests_passed" = True
- "Coverage >= 90%" → checks if context matches regex
- "No lint errors" → checks context for "lint_errors" = 0
The context dict should contain string keys with string or boolean
values representing the current state of the plan/changeset.
"""
def evaluate(
self,
criteria: list[DoDCriterion],
context: dict[str, Any],
) -> list[DoDResult]:
"""Evaluate criteria via text/regex matching.
For each criterion:
- If ``pattern`` is set, use regex matching against each
individual context key and value (same granularity as
substring mode).
- Otherwise, check if any context key/value contains the
criterion text (case-insensitive substring match).
- If the context is empty, all criteria are marked SKIPPED.
Regex patterns longer than ``MAX_REGEX_PATTERN_LENGTH`` are
rejected to mitigate ReDoS attacks.
Args:
criteria: Parsed DoD criteria.
context: Key-value pairs representing plan state.
Returns:
A ``DoDResult`` for each criterion.
"""
results: list[DoDResult] = []
for criterion in criteria:
if not context:
results.append(
DoDResult(
criterion=criterion,
status=DoDStatus.SKIPPED,
reasoning="No evaluation context provided.",
)
)
continue
if criterion.pattern:
results.append(self._evaluate_regex(criterion, context))
else:
results.append(self._evaluate_substring(criterion, context))
return results
# -- private helpers ---------------------------------------------------
@staticmethod
def _evaluate_regex(
criterion: DoDCriterion,
context: dict[str, Any],
) -> DoDResult:
"""Evaluate a single criterion using its regex pattern."""
pattern = criterion.pattern or ""
if len(pattern) > MAX_REGEX_PATTERN_LENGTH:
return DoDResult(
criterion=criterion,
status=DoDStatus.FAILED,
reasoning=(
f"Regex pattern length {len(pattern)} exceeds maximum "
f"{MAX_REGEX_PATTERN_LENGTH}"
),
)
try:
compiled = re.compile(pattern, re.IGNORECASE)
except re.error as exc:
return DoDResult(
criterion=criterion,
status=DoDStatus.FAILED,
reasoning=f"Invalid regex pattern: {exc}",
)
for k, v in context.items():
if compiled.search(str(k)) or compiled.search(str(v)):
return DoDResult(
criterion=criterion,
status=DoDStatus.PASSED,
reasoning=f"Pattern '{pattern}' matched in context.",
)
return DoDResult(
criterion=criterion,
status=DoDStatus.FAILED,
reasoning=(f"Pattern '{pattern}' did not match any context value."),
)
@staticmethod
def _evaluate_substring(
criterion: DoDCriterion,
context: dict[str, Any],
) -> DoDResult:
"""Evaluate a single criterion using substring matching."""
lower_text = criterion.text.lower()
found = any(
lower_text in str(k).lower() or lower_text in str(v).lower()
for k, v in context.items()
)
if found:
return DoDResult(
criterion=criterion,
status=DoDStatus.PASSED,
reasoning="Criterion text found in context.",
)
return DoDResult(
criterion=criterion,
status=DoDStatus.FAILED,
reasoning=("Criterion text not found in any context key or value."),
)