diff --git a/benchmarks/dod_evaluation_bench.py b/benchmarks/dod_evaluation_bench.py new file mode 100644 index 000000000..91ab491a3 --- /dev/null +++ b/benchmarks/dod_evaluation_bench.py @@ -0,0 +1,119 @@ +"""ASV benchmarks for definition-of-done evaluation throughput. + +Measures the performance of: +- DoD text parsing into criteria +- Template rendering with argument substitution +- TextMatchEvaluator evaluation +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +# Ensure the local *source* tree is importable. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.domain.models.core.definition_of_done import ( # noqa: E402 + TextMatchEvaluator, + parse_dod_criteria, + render_dod_template, +) + +# --------------------------------------------------------------------------- +# Benchmark: parse_dod_criteria +# --------------------------------------------------------------------------- + +_SIMPLE_DOD = "All tests pass\nNo lint errors\nCoverage above 90%" +_BULLET_DOD = "- All tests pass\n- No lint errors\n- Coverage above 90%\n- No warnings" +_NUMBERED_DOD = ( + "1. Tests pass\n2. Lint clean\n3. Coverage >= 97%\n4. Types check\n5. Docs updated" +) + + +class ParseDoDCriteria: + """Benchmark suite for ``parse_dod_criteria``.""" + + def time_parse_simple(self) -> None: + """Parse plain-text DoD criteria.""" + parse_dod_criteria(_SIMPLE_DOD) + + def time_parse_bullets(self) -> None: + """Parse bullet-point DoD criteria.""" + parse_dod_criteria(_BULLET_DOD) + + def time_parse_numbered(self) -> None: + """Parse numbered DoD criteria.""" + parse_dod_criteria(_NUMBERED_DOD) + + def time_parse_empty(self) -> None: + """Parse empty DoD text.""" + parse_dod_criteria("") + + +# --------------------------------------------------------------------------- +# Benchmark: render_dod_template +# --------------------------------------------------------------------------- + +_TEMPLATE = "Tests for {module} in {env} must pass with {threshold}% coverage" +_ARGS = {"module": "auth", "env": "production", "threshold": "95"} + + +class RenderDoDTemplate: + """Benchmark suite for ``render_dod_template``.""" + + def time_render_with_args(self) -> None: + """Render template with all placeholders matched.""" + render_dod_template(_TEMPLATE, _ARGS) + + def time_render_partial_args(self) -> None: + """Render template with only some placeholders matched.""" + render_dod_template(_TEMPLATE, {"module": "auth"}) + + def time_render_no_args(self) -> None: + """Render template with empty arguments dict.""" + render_dod_template(_TEMPLATE, {}) + + +# --------------------------------------------------------------------------- +# Benchmark: TextMatchEvaluator +# --------------------------------------------------------------------------- + + +class TextMatchEvaluation: + """Benchmark suite for ``TextMatchEvaluator.evaluate``.""" + + def setup(self) -> None: + """Prepare evaluator and criteria.""" + self.evaluator = TextMatchEvaluator() + self.criteria = parse_dod_criteria(_NUMBERED_DOD) + self.context_all_pass = { + "tests": "tests pass", + "lint": "lint clean", + "coverage": "coverage >= 97%", + "types": "types check", + "docs": "docs updated", + } + self.context_mixed = { + "tests": "tests pass", + "lint": "3 lint errors", + } + + def time_evaluate_all_pass(self) -> None: + """Evaluate criteria where all pass.""" + self.evaluator.evaluate(self.criteria, self.context_all_pass) + + def time_evaluate_mixed_results(self) -> None: + """Evaluate criteria with mixed pass/fail.""" + self.evaluator.evaluate(self.criteria, self.context_mixed) + + def time_evaluate_empty_context(self) -> None: + """Evaluate criteria with empty context (all skip).""" + self.evaluator.evaluate(self.criteria, {}) diff --git a/docs/reference/definition_of_done.md b/docs/reference/definition_of_done.md new file mode 100644 index 000000000..6b927c160 --- /dev/null +++ b/docs/reference/definition_of_done.md @@ -0,0 +1,115 @@ +# Definition-of-Done Gating + +## Overview + +The definition-of-done (DoD) subsystem enforces completion criteria before +a plan is permitted to apply changes. Each plan may carry a +`definition_of_done` text field listing testable criteria that must be +satisfied before the apply phase proceeds. + +## Domain Models + +### `DoDStatus` + +A `StrEnum` with three values: + +| Value | Meaning | +|-----------|------------------------------------------| +| `passed` | Criterion was met | +| `failed` | Criterion was **not** met | +| `skipped` | Criterion could not be evaluated | + +### `DoDCriterion` + +A single testable criterion parsed from DoD text. + +| Field | Type | Description | +|----------|--------------|-------------------------------------------| +| `index` | `int` | Zero-based position in the DoD list | +| `text` | `str` | Human-readable criterion text | +| `pattern`| `str | None` | Optional regex pattern for evaluation | + +### `DoDResult` + +Evaluation result for a single criterion. + +| Field | Type | Description | +|-------------|----------------|--------------------------------------| +| `criterion` | `DoDCriterion` | The evaluated criterion | +| `status` | `DoDStatus` | Pass/fail/skip result | +| `reasoning` | `str` | Explanation for the evaluation | + +### `DoDSummary` + +Aggregated evaluation results for all criteria in a plan. + +| Field | Type | Description | +|---------------------|-------------------|---------------------------------| +| `plan_id` | `str` | Plan ULID | +| `definition_of_done`| `str` | Original DoD text | +| `results` | `list[DoDResult]` | Per-criterion results | +| `evaluated_at` | `datetime` | Evaluation timestamp | + +**Properties:** `total`, `passed`, `failed`, `skipped`, `all_passed`, `is_empty` + +**Methods:** +- `to_cli_dict()` — Dictionary for CLI display +- `to_validation_summary()` — Dict compatible with plan `validation_summary` + +## Functions + +### `parse_dod_criteria(definition_of_done: str) -> list[DoDCriterion]` + +Parses DoD text into individual criteria. Supports: +- Plain text (one criterion per line) +- Bullet points (`-` or `*` prefix) +- Numbered lists (`1.`, `2.` prefix) +- Empty/whitespace-only input returns empty list + +### `render_dod_template(template: str, arguments: dict) -> str` + +Substitutes `{placeholder}` markers in a DoD template with values from +the arguments dict. Unmatched placeholders are preserved as-is. + +## Evaluator Interface + +### `DoDEvaluator` (ABC) + +Abstract base class for DoD evaluation. Implementations must provide: + +```python +def evaluate( + self, + criteria: list[DoDCriterion], + context: dict[str, Any], +) -> list[DoDResult]: ... +``` + +### `TextMatchEvaluator` + +Default evaluator using text/regex matching: + +- If a criterion has a `pattern`, uses regex matching against context values +- Otherwise, checks if criterion text appears in any context key or value +- Empty context causes all criteria to be marked `SKIPPED` +- Invalid regex patterns result in `FAILED` status with error reasoning + +## Integration with Plan Lifecycle + +The DoD evaluation result feeds into the plan's `validation_summary` via +`DoDSummary.to_validation_summary()`, which produces a dict compatible +with the existing validation gate format: + +```python +{ + "total": 3, + "required_passed": 2, + "required_failed": 1, + "informational_passed": 0, + "informational_failed": 0, + "dod_evaluated": True, + "dod_all_passed": False, +} +``` + +When `dod_all_passed` is `False`, the apply phase is blocked. diff --git a/features/definition_of_done.feature b/features/definition_of_done.feature new file mode 100644 index 000000000..a7a79d264 --- /dev/null +++ b/features/definition_of_done.feature @@ -0,0 +1,185 @@ +Feature: Definition-of-Done gating + As a plan executor + I want apply to be blocked unless all DoD criteria are met + So that only plans meeting completion criteria are committed + + Background: + Given a definition of done test environment + + # -- DoD parsing --------------------------------------------------------- + + Scenario: Parse DoD text into criteria + When I parse the DoD text "All tests pass\nNo lint errors\nCoverage above 90%" + Then there should be 3 criteria parsed + And criterion 0 text should be "All tests pass" + And criterion 1 text should be "No lint errors" + + Scenario: Parse DoD with bullet points + When I parse the DoD text "- Tests pass\n- No errors\n* Coverage ok" + Then there should be 3 criteria parsed + And criterion 0 text should be "Tests pass" + + Scenario: Parse DoD with numbered items + When I parse the DoD text "1. Tests pass\n2. No errors" + Then there should be 2 criteria parsed + And criterion 0 text should be "Tests pass" + And criterion 1 text should be "No errors" + + Scenario: Parse empty DoD returns no criteria + When I parse an empty DoD text + Then there should be 0 criteria parsed + + # -- Template rendering -------------------------------------------------- + + Scenario: Render DoD template with plan arguments + When I render the DoD template "Tests for {module} must pass" with argument "module" as "auth" + Then the rendered DoD should be "Tests for auth must pass" + + Scenario: Render DoD template preserves unmatched placeholders + When I render the DoD template "Deploy {service} to {env}" with argument "service" as "api" + Then the rendered DoD should contain "{env}" + + # -- Text match evaluation ----------------------------------------------- + + Scenario: TextMatchEvaluator passes when context contains criterion text + Given DoD criteria from "All tests pass" + And evaluation context with key "status" value "all tests pass" + When I evaluate the criteria with TextMatchEvaluator + Then all criteria should have passed + + Scenario: TextMatchEvaluator fails when context lacks criterion text + Given DoD criteria from "All tests pass" + And evaluation context with key "status" value "3 tests failed" + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "failed" + + Scenario: TextMatchEvaluator with regex pattern passes on match + Given a DoD criterion with text "Coverage check" and pattern "coverage.*9[0-9]" + And evaluation context with key "coverage" value "coverage=95%" + When I evaluate the criteria with TextMatchEvaluator + Then all criteria should have passed + + Scenario: TextMatchEvaluator with regex pattern fails on no match + Given a DoD criterion with text "Coverage check" and pattern "coverage.*9[0-9]" + And evaluation context with key "coverage" value "coverage=80%" + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "failed" + + Scenario: TextMatchEvaluator skips when context is empty + Given DoD criteria from "All tests pass" + And an empty evaluation context + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "skipped" + + Scenario: TextMatchEvaluator handles invalid regex gracefully + Given a DoD criterion with text "Bad regex" and pattern "[invalid" + And evaluation context with key "data" value "anything" + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "failed" + And criterion 0 reasoning should contain "Invalid regex" + + # -- DoDSummary model ---------------------------------------------------- + + Scenario: DoDSummary reports all_passed when no failures + Given a DoDSummary with 2 passed and 0 failed + Then the summary should report all passed + And the DoD summary total should be 2 + + Scenario: DoDSummary reports not all passed when failures exist + Given a DoDSummary with 1 passed and 1 failed + Then the summary should not report all passed + And the summary failed count should be 1 + + Scenario: DoDSummary empty check + Given an empty DoDSummary + Then the summary should be empty + And the summary should not report all passed + + Scenario: DoDSummary to_cli_dict has expected fields + Given a DoDSummary with 1 passed and 0 failed + Then the summary CLI dict should have key "total" + And the summary CLI dict should have key "all_passed" + And the summary CLI dict should have key "criteria" + + Scenario: DoDSummary to_validation_summary is compatible + Given a DoDSummary with 2 passed and 1 failed + Then the validation summary should have required_passed 2 + And the validation summary should have required_failed 1 + And the validation summary should have dod_evaluated true + + # -- Evaluation context matching ----------------------------------------- + + Scenario: TextMatchEvaluator matches criterion text in context keys + Given DoD criteria from "lint_errors" + And evaluation context with key "lint_errors" value "0" + When I evaluate the criteria with TextMatchEvaluator + Then all criteria should have passed + + Scenario: Multiple criteria with mixed results + Given DoD criteria from "tests pass\nlint errors" + And evaluation context with key "result" value "tests pass, no issues" + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "passed" + And criterion 1 should have status "failed" + + # -- Coverage: edge cases ------------------------------------------------ + + Scenario: Parse DoD with blank lines between criteria + When I parse the DoD text "Tests pass\n\nNo errors" + Then there should be 2 criteria parsed + + Scenario: Parse DoD with whitespace only line + When I parse the DoD text " \n- Tests pass" + Then there should be 1 criteria parsed + And criterion 0 text should be "Tests pass" + + Scenario: TextMatchEvaluator matches via individual key lookup + Given DoD criteria from "all_tests_passed" + And evaluation context with key "all_tests_passed" value "true" + When I evaluate the criteria with TextMatchEvaluator + Then all criteria should have passed + + # -- T1: all-SKIPPED summary gate behavior -------------------------------- + + Scenario: DoDSummary all_passed is false when all criteria skipped + Given a DoDSummary with all criteria skipped + Then the summary should not report all passed + And the summary skipped count should be 2 + + # -- T2: bullet prefix yielding empty text -------------------------------- + + Scenario: Parse DoD with bullet prefix but no text + When I parse the DoD text "- \n- Tests pass" + Then there should be 1 criteria parsed + And criterion 0 text should be "Tests pass" + + # -- T5: multi-argument template rendering -------------------------------- + + Scenario: Render DoD template with multiple arguments + When I render the DoD template "{svc} in {env} OK" with arguments "svc"="api" and "env"="prod" + Then the rendered DoD should be "api in prod OK" + + Scenario: Render DoD template prevents injection via values + When I render the DoD template "Test {name}" with argument "name" as "{secret}" + Then the rendered DoD should be "Test {secret}" + + # -- S3: length limit on template ----------------------------------------- + + Scenario: Render DoD rejects oversized template + Given a DoD template that is 11000 characters long + When I render the oversized DoD template with argument "x" as "y" + Then the DoD render should fail with "exceeds maximum" + + Scenario: Render DoD rejects oversized output + Given a short DoD template with a huge argument value + When I render the DoD template that produces oversized output + Then the DoD render should fail with "exceeds maximum" + + # -- S2: ReDoS protection ------------------------------------------------- + + Scenario: TextMatchEvaluator rejects oversized regex pattern + Given a DoD criterion with text "long regex" and pattern that is 600 chars + And evaluation context with key "data" value "anything" + When I evaluate the criteria with TextMatchEvaluator + Then criterion 0 should have status "failed" + And criterion 0 reasoning should contain "exceeds maximum" diff --git a/features/steps/definition_of_done_steps.py b/features/steps/definition_of_done_steps.py new file mode 100644 index 000000000..046c1e741 --- /dev/null +++ b/features/steps/definition_of_done_steps.py @@ -0,0 +1,327 @@ +"""Step definitions for definition-of-done gating tests.""" + +from __future__ import annotations + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.domain.models.core.definition_of_done import ( + DoDCriterion, + DoDResult, + DoDStatus, + DoDSummary, + TextMatchEvaluator, + parse_dod_criteria, + render_dod_template, +) + +__all__: list[str] = [] + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a definition of done test environment") +def step_given_dod_env(context: Context) -> None: + context.dod_criteria = None # list[DoDCriterion] | None + context.dod_rendered = None # str | None + context.dod_results = None # list[DoDResult] | None + context.dod_context = {} # dict[str, Any] + context.dod_summary = None # DoDSummary | None + + +# --------------------------------------------------------------------------- +# Given steps +# --------------------------------------------------------------------------- + + +@given('DoD criteria from "{text}"') +def step_given_criteria_from(context: Context, text: str) -> None: + resolved = text.replace("\\n", "\n") + context.dod_criteria = parse_dod_criteria(resolved) + + +@given('evaluation context with key "{key}" value "{value}"') +def step_given_eval_context(context: Context, key: str, value: str) -> None: + context.dod_context[key] = value + + +@given("an empty evaluation context") +def step_given_empty_context(context: Context) -> None: + context.dod_context = {} + + +@given('a DoD criterion with text "{text}" and pattern "{pattern}"') +def step_given_criterion_with_pattern( + context: Context, text: str, pattern: str +) -> None: + context.dod_criteria = [DoDCriterion(index=0, text=text, pattern=pattern)] + + +@given("a DoDSummary with {p} passed and {f} failed") +def step_given_summary(context: Context, p: str, f: str) -> None: + results: list[DoDResult] = [] + for i in range(int(p)): + results.append( + DoDResult( + criterion=DoDCriterion(index=i, text=f"crit-{i}"), + status=DoDStatus.PASSED, + reasoning="ok", + ) + ) + for i in range(int(f)): + results.append( + DoDResult( + criterion=DoDCriterion(index=int(p) + i, text=f"fail-{i}"), + status=DoDStatus.FAILED, + reasoning="not ok", + ) + ) + context.dod_summary = DoDSummary( + plan_id="01DODTEST0000000000001", + definition_of_done="test dod", + results=results, + ) + + +@given("an empty DoDSummary") +def step_given_empty_summary(context: Context) -> None: + context.dod_summary = DoDSummary( + plan_id="01DODTEST0000000000001", + definition_of_done="empty", + results=[], + ) + + +@given("a DoDSummary with all criteria skipped") +def step_given_all_skipped_summary(context: Context) -> None: + results = [ + DoDResult( + criterion=DoDCriterion(index=i, text=f"skip-{i}"), + status=DoDStatus.SKIPPED, + reasoning="skipped", + ) + for i in range(2) + ] + context.dod_summary = DoDSummary( + plan_id="01DODTEST0000000000001", + definition_of_done="skip test", + results=results, + ) + + +@given("a DoD template that is {n} characters long") +def step_given_long_dod_template(context: Context, n: str) -> None: + context.dod_oversized_template = "x" * int(n) + + +@given('a DoD criterion with text "{text}" and pattern that is {n} chars') +def step_given_long_pattern(context: Context, text: str, n: str) -> None: + context.dod_criteria = [ + DoDCriterion(index=0, text=text, pattern="a" * int(n)), + ] + + +@given("a short DoD template with a huge argument value") +def step_given_huge_value_template(context: Context) -> None: + context.dod_short_template = "Result: {val}" + context.dod_huge_value = "x" * 60_000 + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when('I parse the DoD text "{text}"') +def step_when_parse(context: Context, text: str) -> None: + resolved = text.replace("\\n", "\n") + context.dod_criteria = parse_dod_criteria(resolved) + + +@when("I parse an empty DoD text") +def step_when_parse_empty(context: Context) -> None: + context.dod_criteria = parse_dod_criteria("") + + +@when('I render the DoD template "{template}" with argument "{key}" as "{value}"') +def step_when_render(context: Context, template: str, key: str, value: str) -> None: + context.dod_rendered = render_dod_template(template, {key: value}) + + +@when( + 'I render the DoD template "{template}" with arguments "{k1}"="{v1}" and "{k2}"="{v2}"' +) +def step_when_render_multi( + context: Context, template: str, k1: str, v1: str, k2: str, v2: str +) -> None: + context.dod_rendered = render_dod_template(template, {k1: v1, k2: v2}) + + +@when('I render the oversized DoD template with argument "{key}" as "{value}"') +def step_when_render_oversized(context: Context, key: str, value: str) -> None: + try: + context.dod_rendered = render_dod_template( + context.dod_oversized_template, {key: value} + ) + context.dod_render_error = None + except ValueError as exc: + context.dod_render_error = exc + + +@when("I render the DoD template that produces oversized output") +def step_when_render_oversized_output(context: Context) -> None: + try: + context.dod_rendered = render_dod_template( + context.dod_short_template, {"val": context.dod_huge_value} + ) + context.dod_render_error = None + except ValueError as exc: + context.dod_render_error = exc + + +@when("I evaluate the criteria with TextMatchEvaluator") +def step_when_evaluate(context: Context) -> None: + evaluator = TextMatchEvaluator() + context.dod_results = evaluator.evaluate(context.dod_criteria, context.dod_context) + + +# --------------------------------------------------------------------------- +# Then steps +# --------------------------------------------------------------------------- + + +@then("there should be {n} criteria parsed") +def step_then_criteria_count(context: Context, n: str) -> None: + assert context.dod_criteria is not None + assert len(context.dod_criteria) == int(n), ( + f"Expected {n} criteria, got {len(context.dod_criteria)}" + ) + + +@then('criterion {i} text should be "{text}"') +def step_then_criterion_text(context: Context, i: str, text: str) -> None: + assert context.dod_criteria is not None + idx = int(i) + assert idx < len(context.dod_criteria) + assert context.dod_criteria[idx].text == text, ( + f"Expected text '{text}', got '{context.dod_criteria[idx].text}'" + ) + + +@then('the rendered DoD should be "{expected}"') +def step_then_rendered(context: Context, expected: str) -> None: + assert context.dod_rendered == expected, ( + f"Expected '{expected}', got '{context.dod_rendered}'" + ) + + +@then('the rendered DoD should contain "{text}"') +def step_then_rendered_contains(context: Context, text: str) -> None: + assert context.dod_rendered is not None + assert text in context.dod_rendered + + +@then("all criteria should have passed") +def step_then_all_passed(context: Context) -> None: + assert context.dod_results is not None + for r in context.dod_results: + assert r.status == DoDStatus.PASSED, ( + f"Criterion '{r.criterion.text}' has status '{r.status.value}'" + ) + + +@then('criterion {i} should have status "{status}"') +def step_then_criterion_status(context: Context, i: str, status: str) -> None: + assert context.dod_results is not None + idx = int(i) + assert idx < len(context.dod_results) + assert context.dod_results[idx].status == DoDStatus(status), ( + f"Expected status '{status}', got '{context.dod_results[idx].status.value}'" + ) + + +@then('criterion {i} reasoning should contain "{text}"') +def step_then_criterion_reasoning(context: Context, i: str, text: str) -> None: + assert context.dod_results is not None + idx = int(i) + assert text in context.dod_results[idx].reasoning, ( + f"Expected reasoning to contain '{text}', " + f"got: {context.dod_results[idx].reasoning}" + ) + + +@then("the summary should report all passed") +def step_then_summary_all_passed(context: Context) -> None: + assert context.dod_summary is not None + assert context.dod_summary.all_passed + + +@then("the summary should not report all passed") +def step_then_summary_not_all_passed(context: Context) -> None: + assert context.dod_summary is not None + assert not context.dod_summary.all_passed + + +@then("the DoD summary total should be {n}") +def step_then_dod_summary_total(context: Context, n: str) -> None: + assert context.dod_summary is not None + assert context.dod_summary.total == int(n) + + +@then("the summary failed count should be {n}") +def step_then_summary_failed(context: Context, n: str) -> None: + assert context.dod_summary is not None + assert context.dod_summary.failed == int(n) + + +@then("the summary should be empty") +def step_then_summary_empty(context: Context) -> None: + assert context.dod_summary is not None + assert context.dod_summary.is_empty + + +@then('the summary CLI dict should have key "{key}"') +def step_then_cli_dict_key(context: Context, key: str) -> None: + assert context.dod_summary is not None + cli_dict = context.dod_summary.to_cli_dict() + assert key in cli_dict + + +@then("the validation summary should have required_passed {n}") +def step_then_val_summary_passed(context: Context, n: str) -> None: + assert context.dod_summary is not None + vs = context.dod_summary.to_validation_summary() + assert vs["required_passed"] == int(n) + + +@then("the validation summary should have required_failed {n}") +def step_then_val_summary_failed(context: Context, n: str) -> None: + assert context.dod_summary is not None + vs = context.dod_summary.to_validation_summary() + assert vs["required_failed"] == int(n) + + +@then("the validation summary should have dod_evaluated true") +def step_then_val_summary_dod(context: Context) -> None: + assert context.dod_summary is not None + vs = context.dod_summary.to_validation_summary() + assert vs["dod_evaluated"] is True + + +@then("the summary skipped count should be {n}") +def step_then_summary_skipped(context: Context, n: str) -> None: + assert context.dod_summary is not None + assert context.dod_summary.skipped == int(n), ( + f"Expected skipped={n}, got {context.dod_summary.skipped}" + ) + + +@then('the DoD render should fail with "{text}"') +def step_then_dod_render_error(context: Context, text: str) -> None: + assert context.dod_render_error is not None, "Expected a ValueError" + assert text in str(context.dod_render_error), ( + f"Expected '{text}' in error: {context.dod_render_error}" + ) diff --git a/robot/definition_of_done.robot b/robot/definition_of_done.robot new file mode 100644 index 000000000..c0853acd9 --- /dev/null +++ b/robot/definition_of_done.robot @@ -0,0 +1,100 @@ +*** Settings *** +Documentation Integration tests for definition-of-done gating +Resource common.resource +Library Process +Library OperatingSystem +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Test Cases *** +Parse DoD Text Into Criteria + [Documentation] Verify DoD text is parsed into individual criteria + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... parse All tests pass\\nNo lint errors\\nCoverage above 90% + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} dod-parse-ok: 3 criteria + Should Contain ${result.stdout} All tests pass + +Render DoD Template With Arguments + [Documentation] Verify DoD template rendering substitutes placeholders + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... render Tests for {module} must pass module auth + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} dod-render-ok: Tests for auth must pass + +Evaluate DoD Criteria Against Context + [Documentation] Verify TextMatchEvaluator evaluates criteria correctly + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... evaluate All tests pass status all tests pass + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} dod-evaluate-ok: 1 passed, 0 failed + +Build DoDSummary CLI Dict + [Documentation] Verify DoDSummary produces correct CLI dict output + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... summary 01PLANTEST0000000000001 test dod 2 1 + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} dod-summary-ok: + Should Contain ${result.stdout} all_passed + +Build DoDSummary Validation Summary + [Documentation] Verify DoDSummary produces compatible validation summary + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... validate-summary 01PLANTEST0000000000001 test dod 3 0 + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} dod-validate-summary-ok: + Should Contain ${result.stdout} dod_evaluated + +Helper Shows Usage On No Command + [Documentation] Verify helper prints usage when no command given + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage + +Helper Rejects Unknown Command + [Documentation] Verify helper rejects an unknown sub-command + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... bogus stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Unknown command + +Parse Shows Usage On No Args + [Documentation] Verify parse sub-command prints usage with no arguments + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... parse stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage + +Render Shows Usage On Insufficient Args + [Documentation] Verify render sub-command prints usage with insufficient args + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... render template_only stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage + +Evaluate Shows Usage On Insufficient Args + [Documentation] Verify evaluate sub-command prints usage with insufficient args + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... evaluate dod_only stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage + +Summary Shows Usage On Insufficient Args + [Documentation] Verify summary sub-command prints usage with insufficient args + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... summary plan_id_only stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage + +Validate Summary Shows Usage On Insufficient Args + [Documentation] Verify validate-summary sub-command prints usage with insufficient args + ${result}= Run Process ${PYTHON} ${CURDIR}/helper_definition_of_done.py + ... validate-summary plan_id_only stderr=STDOUT + Should Be Equal As Integers ${result.rc} 1 + Should Contain ${result.stdout} Usage diff --git a/robot/helper_definition_of_done.py b/robot/helper_definition_of_done.py new file mode 100644 index 000000000..8d9963255 --- /dev/null +++ b/robot/helper_definition_of_done.py @@ -0,0 +1,175 @@ +"""Robot Framework helper for definition-of-done evaluation. + +Provides a CLI-style interface for Robot to invoke DoD parsing, +template rendering, and evaluation. Exit code 0 = success, 1 = failure. + +Usage: + python robot/helper_definition_of_done.py parse + python robot/helper_definition_of_done.py render