Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4a969eac77 | |||
| 4db5a06e0a | |||
| 9cfa1dd1d7 | |||
| 3b83438e7d | |||
| 1e385b8f61 | |||
| f37bfa01fe | |||
| d82f5de2eb | |||
| 39bbff0849 | |||
| 52830971f2 | |||
|
9fe69c468c
|
|||
| d25a060c58 |
@@ -285,76 +285,6 @@ jobs:
|
||||
path: build/nox-integration-tests-output.log
|
||||
retention-days: 30
|
||||
|
||||
tdd_quality_gate:
|
||||
if: forgejo.event_name == 'pull_request'
|
||||
runs-on: docker
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for diff analysis)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Fetch PR base branch for diff analysis
|
||||
run: |
|
||||
git fetch origin "${{ forgejo.base_ref }}"
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-tdd-gate-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-tdd-gate-
|
||||
|
||||
- name: Run TDD quality gate via nox
|
||||
run: |
|
||||
nox -s tdd_quality_gate
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
PR_DESCRIPTION: ${{ forgejo.event.pull_request.body }}
|
||||
PR_BASE_REF: ${{ forgejo.base_ref }}
|
||||
|
||||
e2e_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for E2E tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-tests-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-tests-
|
||||
|
||||
- name: Run E2E tests via nox
|
||||
run: |
|
||||
nox -s e2e_tests
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
coverage:
|
||||
runs-on: docker
|
||||
container:
|
||||
@@ -627,7 +557,7 @@ jobs:
|
||||
echo "=== Push access smoke-test passed ==="
|
||||
status-check:
|
||||
if: always()
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, coverage, build, docker, helm, push-validation, tdd_quality_gate]
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, coverage, build, docker, helm, push-validation]
|
||||
runs-on: docker
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
@@ -645,7 +575,6 @@ jobs:
|
||||
echo "docker: ${{ needs.docker.result }}"
|
||||
echo "helm: ${{ needs.helm.result }}"
|
||||
echo "push-validation: ${{ needs.push-validation.result }}"
|
||||
echo "tdd_quality_gate: ${{ needs.tdd_quality_gate.result }}"
|
||||
|
||||
if [ "${{ needs.lint.result }}" != "success" ] || \
|
||||
[ "${{ needs.typecheck.result }}" != "success" ] || \
|
||||
@@ -661,10 +590,4 @@ jobs:
|
||||
echo "FAILED: One or more required jobs did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${{ forgejo.event_name }}" = "pull_request" ] && \
|
||||
[ "${{ needs.tdd_quality_gate.result }}" != "success" ]; then
|
||||
echo "FAILED: tdd_quality_gate did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
echo "All required CI checks passed"
|
||||
|
||||
@@ -136,6 +136,18 @@ Entered when `review_type` is `ci_flag`. This is a lightweight review that only
|
||||
|
||||
Entered when `review_type` is `re_review`. The PR previously received `REQUEST_CHANGES` feedback and the author has since pushed new commits. Your job is to verify the feedback was adequately addressed AND conduct a full review of the current state.
|
||||
|
||||
0. **Post review-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{pr_number}/comments` with body:
|
||||
```
|
||||
> Starting review for PR #{pr_number} (`re_review`)...
|
||||
>
|
||||
> Previous feedback was addressed — verifying changes and conducting full re-review. This may take a few minutes.
|
||||
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: PR Review | Agent: pr-review-worker
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
1. **Read the PR.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{pr_number}` — read title, body, labels, milestone, and linked issues.
|
||||
|
||||
2. **Read all linked issues.** Parse `Closes #N`, `Fixes #N`, and `Refs #N` patterns from the PR body. For each referenced issue, GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{issue_number}` — read title, body, labels, and acceptance criteria.
|
||||
@@ -176,6 +188,18 @@ Entered when `review_type` is `re_review`. The PR previously received `REQUEST_C
|
||||
|
||||
Entered when `review_type` is `first_review`. The PR has no active review feedback — this is a fresh evaluation from scratch.
|
||||
|
||||
0. **Post review-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{pr_number}/comments` with body:
|
||||
```
|
||||
> Starting review for PR #{pr_number} (`first_review`)...
|
||||
>
|
||||
> Conducting fresh evaluation against the full review checklist. This may take a few minutes.
|
||||
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: PR Review | Agent: pr-review-worker
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
1. **Read the PR.** Same as Re-Review Mode step 1.
|
||||
|
||||
2. **Read all linked issues.** Same as Re-Review Mode step 2.
|
||||
|
||||
@@ -121,16 +121,28 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
|
||||
1. **Read the issue.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}` — read title, body, labels, milestone, and metadata section (branch name, commit message format). Paginate all comments to understand full context and any subtask structure.
|
||||
|
||||
2. **Determine branch name.** Extract the branch name from the issue's Metadata section if present. If absent, derive one: `feature/issue-{work_number}-{kebab-slug-of-title}`.
|
||||
2. **Post work-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments` with body:
|
||||
```
|
||||
> Starting implementation for issue #{work_number}: {issue_title} (`issue_impl`)...
|
||||
>
|
||||
> Reading requirements and setting up development environment. This may take several minutes.
|
||||
|
||||
3. **Create isolated clone.** Call `git-isolator-util` with `create_branch: true`, `base_branch: master`, and the determined `branch_name` (see Subagents section for prompt template).
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: Implementation | Agent: task-implementor
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Replace `{issue_title}` with the title read in step 1. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
4. **Implement the code.** Load the `cleverthis-guidelines` skill for CONTRIBUTING.md rules and follow them strictly. Key rules:
|
||||
3. **Determine branch name.** Extract the branch name from the issue's Metadata section if present. If absent, derive one: `feature/issue-{work_number}-{kebab-slug-of-title}`.
|
||||
|
||||
4. **Create isolated clone.** Call `git-isolator-util` with `create_branch: true`, `base_branch: master`, and the determined `branch_name` (see Subagents section for prompt template).
|
||||
|
||||
5. **Implement the code.** Load the `cleverthis-guidelines` skill for CONTRIBUTING.md rules and follow them strictly. Key rules:
|
||||
- Source in `src/cleveragents/`, Behave unit tests in `features/`, Robot Framework integration/e2e tests in `robot/`
|
||||
- Full static typing throughout — no `# type: ignore`
|
||||
- All commands via `nox` — never invoke `pip`, `pytest`, `behave`, or `robot` directly
|
||||
|
||||
5. **Run quality gates in order:**
|
||||
6. **Run quality gates in order:**
|
||||
```bash
|
||||
nox -e lint
|
||||
nox -e typecheck
|
||||
@@ -140,11 +152,11 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
nox -e coverage_report
|
||||
```
|
||||
|
||||
6. **Fix any failures.** If a gate fails, fix the code and re-run the failing gate (and any that follow it). Repeat until all gates pass. Do not move forward with failing gates.
|
||||
7. **Fix any failures.** If a gate fails, fix the code and re-run the failing gate (and any that follow it). Repeat until all gates pass. Do not move forward with failing gates.
|
||||
|
||||
7. **Commit.** Call `git-commit-util` with `commit_and_push` operation. The first line of the commit message must match the issue's Metadata section exactly (see Subagents section).
|
||||
8. **Commit.** Call `git-commit-util` with `commit_and_push` operation. The first line of the commit message must match the issue's Metadata section exactly (see Subagents section).
|
||||
|
||||
8. **Create PR.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls` with:
|
||||
9. **Create PR.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls` with:
|
||||
- `title`: taken from the issue title or the commit message first line
|
||||
- `body`: description of changes + `Closes #{work_number}` + dependency link (`This PR blocks issue #{work_number}`)
|
||||
- `base`: `master`
|
||||
@@ -152,35 +164,47 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
- `milestone` (if set on the issue): same milestone ID
|
||||
- Use PAT authentication: `Authorization: token {forgejo_pat}`
|
||||
|
||||
9. **Post attempt comment** on the issue (see "Attempt Comments" section below).
|
||||
10. **Post attempt comment** on the issue (see "Attempt Comments" section below).
|
||||
|
||||
10. **Clean up.** `rm -rf {repo_dir}`
|
||||
11. **Clean up.** `rm -rf {repo_dir}`
|
||||
|
||||
11. **Exit.**
|
||||
12. **Exit.**
|
||||
|
||||
#### Procedure: `pr_fix` (PR Fix)
|
||||
|
||||
1. **Read the PR.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}` — read description, head branch, head SHA, and CI state. Set `branch_name` to the PR's head branch.
|
||||
|
||||
2. **Read all reviews.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}/reviews?limit=50&page=N` — paginate fully. For any review in `REQUEST_CHANGES` state, GET its comments to understand the specific feedback.
|
||||
2. **Post work-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments` with body:
|
||||
```
|
||||
> Starting fix for PR #{work_number}: {pr_title} (`pr_fix`)...
|
||||
>
|
||||
> Addressing review feedback and CI failures. This may take several minutes.
|
||||
|
||||
3. **Read all PR comments.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments?limit=50&page=N` — paginate fully.
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: Implementation | Agent: task-implementor
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Replace `{pr_title}` with the title read in step 1. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
4. **Fetch CI failure details.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/commits/{head_sha}/statuses?limit=50&page=N` — paginate fully. For each failing status that has a `target_url`, webfetch that URL to retrieve the failure logs.
|
||||
3. **Read all reviews.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}/reviews?limit=50&page=N` — paginate fully. For any review in `REQUEST_CHANGES` state, GET its comments to understand the specific feedback.
|
||||
|
||||
5. **Create isolated clone.** Call `git-isolator-util` with `create_branch: false` and `branch: {branch_name}` (the PR's head branch, resolved in step 1).
|
||||
4. **Read all PR comments.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments?limit=50&page=N` — paginate fully.
|
||||
|
||||
6. **Fix the issues.** Address all CI failures and all unresolved reviewer feedback. Never partially address reviewer comments — every `REQUEST_CHANGES` concern must be fully resolved.
|
||||
5. **Fetch CI failure details.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/commits/{head_sha}/statuses?limit=50&page=N` — paginate fully. For each failing status that has a `target_url`, webfetch that URL to retrieve the failure logs.
|
||||
|
||||
7. **Run quality gates locally** (same 6 gates as above). All must pass before pushing. Fix and re-run as many times as needed.
|
||||
6. **Create isolated clone.** Call `git-isolator-util` with `create_branch: false` and `branch: {branch_name}` (the PR's head branch, resolved in step 1).
|
||||
|
||||
8. **Commit and push.** Call `git-commit-util` with `force_push_with_lease` operation (see Subagents section).
|
||||
7. **Fix the issues.** Address all CI failures and all unresolved reviewer feedback. Never partially address reviewer comments — every `REQUEST_CHANGES` concern must be fully resolved.
|
||||
|
||||
9. **Post attempt comment** on the PR (see "Attempt Comments" section below).
|
||||
8. **Run quality gates locally** (same 6 gates as above). All must pass before pushing. Fix and re-run as many times as needed.
|
||||
|
||||
10. **Clean up.** `rm -rf {repo_dir}`
|
||||
9. **Commit and push.** Call `git-commit-util` with `force_push_with_lease` operation (see Subagents section).
|
||||
|
||||
11. **Exit.**
|
||||
10. **Post attempt comment** on the PR (see "Attempt Comments" section below).
|
||||
|
||||
11. **Clean up.** `rm -rf {repo_dir}`
|
||||
|
||||
12. **Exit.**
|
||||
|
||||
### Attempt Comments
|
||||
|
||||
|
||||
+44
-50
@@ -1,58 +1,20 @@
|
||||
# Changelog
|
||||
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Hardened the TDD bug-fix quality gate for issue #629: PR parsing now
|
||||
requires whole-word closing keywords (avoids false positives like
|
||||
"prefixes #12"), TDD bug tag discovery now uses exact token matching
|
||||
(avoids `@tdd_bug_42` matching `@tdd_bug_420`), and gate evaluation now
|
||||
requires expected-fail tag removal to be present in the PR diff. CI
|
||||
integration now fetches the PR base branch for diff analysis and includes
|
||||
`tdd_quality_gate` in `status-check` requirements for pull requests.
|
||||
Review-round fixes: `check_expected_fail_removed` now uses word-boundary
|
||||
matching via `_contains_tag_token` (avoids false positives on partial tag
|
||||
names); diff expected-fail removal detection now tracks flags at file level
|
||||
instead of per-hunk (fixes false negatives when tags span different hunks);
|
||||
`parse_bug_refs` filters out issue number zero; redundant double error
|
||||
reporting eliminated; regex compilation cached via `lru_cache`; nox session
|
||||
no longer installs the full project (script uses stdlib only); CI checkout
|
||||
uses `fetch-depth: 0` for reliable merge-base resolution.
|
||||
Review-round 2 fixes: diff expected-fail removal detection now requires the
|
||||
removed line to contain both the expected-fail tag and the specific bug tag
|
||||
(fixes false positives when two bugs' TDD tests reside in the same file);
|
||||
`check_expected_fail_removed` error messages now use the correct tag prefix
|
||||
per file type (`@tdd_bug_N` for `.feature`, `tdd_bug_N` for `.robot`);
|
||||
`bool` values are now rejected by bug-number validation guards; file-read
|
||||
error handling in `find_tdd_tests` and `check_expected_fail_removed` now
|
||||
catches `UnicodeDecodeError` (root-safe unreadable-file handling); temp
|
||||
directory cleanup added to `after_scenario` hook; 8 new Behave scenarios
|
||||
covering bool guards, co-located bug false-positive, `run_quality_gate`
|
||||
argument validation, and `main()` CLI entry point.
|
||||
Review-round 3 fixes: synthetic PR diff helper now auto-detects `.robot`
|
||||
vs `.feature` file type and generates the matching diff format (fixes
|
||||
under-tested robot-format diff code path); `check_expected_fail_removed`
|
||||
test step now filters files by bug tag via `find_tdd_tests` before
|
||||
checking (matches the production code path); `after_scenario` temp
|
||||
directory cleanup no longer sets `context.temp_dir = None` (fixes
|
||||
cleanup conflict with `cli_init_yes_flag_steps.py`); 2 new Behave
|
||||
scenarios covering multi-line PR description parsing and non-string
|
||||
`pr_diff` type guard.
|
||||
- Added Fix-then-Revalidate orchestration loop for required validations:
|
||||
bounded retry with configurable limits (0--100 per Safety Profile),
|
||||
strategy revision escalation via ``auto_strategy_revision`` float
|
||||
threshold, user escalation via ``needs_user_escalation`` result flag,
|
||||
and domain events (``VALIDATION_FIX_ATTEMPTED``, ``VALIDATION_FIX_SUCCEEDED``,
|
||||
``VALIDATION_FIX_EXHAUSTED``). Validation errors are treated as required
|
||||
failures regardless of mode. Includes ``auto_validation_fix`` threshold,
|
||||
per-resource retry tracking, early-exit signalling via ``None`` return from
|
||||
``FixCallback``, event bus circuit breaker with lock-protected failure
|
||||
counter, spec-required ``validation_summary`` and
|
||||
``final_validation_results`` fields on the result model, DI container
|
||||
All notable changes to this project will be documented in this file.
|
||||
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- **Structural Component Output Validation** (#11147): Replaces exact character matching with structural component checking for output validation. Implements three validators covering plan tree output, decision CLI dicts, and structured session snapshots. The `validate_plan_tree` function validates node dicts for required keys (`decision_id`, `type`, `sequence`, `question`, `children`), ULID format, correct types, and sibling ordering. The `validate_decision_dict` function validates decision CLI output against the `Decision.as_cli_dict()` schema with field presence, type, ULID pattern, confidence range [0..1], and boolean field checks. The `validate_structured_output` function validates the StructuredOutput envelope for `command`, `session_id` (ULID), status membership, `exit_code`, and elements integrity. A unified dispatcher (`validate_structured_component_output`) enables routing by target_type. BDD test coverage added in `features/structural_validation.feature`. [Epic #8137](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/8137)
|
||||
|
||||
- **`task-implementor` posts work-started notification comments** (#11031): Both
|
||||
the `issue_impl` and `pr_fix` procedures now post an informational "work
|
||||
started" comment to the Forgejo issue/PR before beginning implementation.
|
||||
The comment includes the issue/PR title, procedure type, and expected
|
||||
duration. Posted asynchronously ("fire and move on") so it does not block
|
||||
the workflow. Step numbering in both procedures has been re-numbered to
|
||||
accommodate the new step.
|
||||
|
||||
- **`agents session tell` invokes real LLM orchestrator actor** (#5784): Replaced the
|
||||
M3 echo-stub with real actor invocation via `SessionWorkflow`, routing through
|
||||
`LangChainSessionCaller` → `ToolCallingRuntime.run_tool_loop()`. The user prompt
|
||||
@@ -74,9 +36,24 @@
|
||||
|
||||
### Added
|
||||
|
||||
- **`pr-review-worker` review-started notification** (#11028): The `first_review`
|
||||
and `re_review` modes now post a "review started" notification comment to the
|
||||
PR at the beginning of the review, giving PR authors immediate visibility
|
||||
that a review is in progress. Posted asynchronously so it does not block
|
||||
the workflow.
|
||||
|
||||
- **Plan Rollback Command** (#8557): Implemented `agents plan rollback <plan-id> [<checkpoint-id>]` for checkpoint-based plan state restoration in Epic #8493. The command restores a plan's sandbox to the state captured at a given checkpoint, discarding all decisions made after that checkpoint. The checkpoint can be specified as an optional positional second argument or via the `--to-checkpoint` named option. Supports `--yes/-y` flag to skip confirmation prompts and `--format/-f` for output format selection (rich/plain/json/yaml). Included with comprehensive BDD test coverage (>= 97%) and spec-aligned output formatting showing rollback summary, changes reverted, impact analysis, and post-rollback state panels.
|
||||
|
||||
### Fixed
|
||||
- **Guard cleanup_stale against execute/processing and execute/complete plans** (#11121):
|
||||
``_create_sandbox_for_plan()`` in ``src/cleveragents/cli/commands/plan.py`` now
|
||||
skips ``GitWorktreeSandbox.cleanup_stale()`` when the plan is in
|
||||
``execute/processing`` (execution in progress) or ``execute/complete`` (execution
|
||||
finished, awaiting apply) state. Previously, re-invoking ``agents plan execute``
|
||||
on a completed plan would silently destroy the ``cleveragents/plan-<id>`` git
|
||||
worktree branch, causing ``plan apply`` to merge zero artifacts. The guard
|
||||
preserves the branch per spec (§sandbox.cleanup defaults to ``on_apply``).
|
||||
|
||||
- **Global CLI options ``--data-dir``, ``--config-path``, and ``-v`` now work correctly**
|
||||
(#6785): These spec-required flags were absent from ``main_callback()`` in
|
||||
``src/cleveragents/cli/main.py``, causing any invocation with these flags to crash
|
||||
@@ -86,6 +63,13 @@
|
||||
sets ``CLEVERAGENTS_CONFIG_PATH`` for ``ConfigService`` to pick up, and ``-v``
|
||||
(repeatable count) maps to the appropriate ``structlog`` log level.
|
||||
|
||||
- **Add regression test for ActorRegistry.add() nested provider/model extraction** (#4321):
|
||||
Added BDD regression test confirming that provider and model are correctly
|
||||
extracted from nested `actors.<name>.config` blocks when `type` is only at
|
||||
the nested level and `name` uses the `local/` namespace prefix. This
|
||||
scenario was previously fixed in #4300; the test ensures the fix remains
|
||||
in place and prevents future regressions.
|
||||
|
||||
- **Actor configuration validation incorrectly requires top-level provider field** (#4300):
|
||||
Actor configuration in V3 is now obtained from the nested configuration
|
||||
parameter, according to the specification.
|
||||
@@ -327,6 +311,16 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`invariant_enforced` decisions not propagated to child plans on subplan spawn** (#9131):
|
||||
Fixed `SubplanService.spawn()` to propagate all `invariant_enforced` decisions from the
|
||||
parent plan's decision tree to each child plan's decision tree. Previously, child plans
|
||||
started Strategize with a completely empty invariant set, violating the spec requirement:
|
||||
"recorded as `invariant_enforced` decisions that propagate to child plans." The fix adds
|
||||
a `_propagate_invariant_decisions()` helper that re-records each parent
|
||||
`invariant_enforced` decision on the child plan, including `non_overridable` global
|
||||
invariants. BDD regression coverage added in
|
||||
`features/tdd_invariant_propagation_subplan.feature`.
|
||||
|
||||
- **fix(repositories): derive PlanResult.success from result_success column instead of error_message** (#7501):
|
||||
Fixed a critical bug in `PlanRepository._to_domain` where `PlanResult.success` was incorrectly
|
||||
derived from `error_message is None`. Because `error_message` is shared between the build phase
|
||||
|
||||
@@ -1236,7 +1236,6 @@ session is missing required tooling, add the dependency to the session before re
|
||||
- **All tests (including static checks):** `nox`
|
||||
- **Unit tests only:** `nox -s unit_tests`
|
||||
- **Integration tests:** `nox -s integration_tests`
|
||||
- **TDD bug-fix quality gate:** `PR_DESCRIPTION="Fixes #123" PR_BASE_REF=master nox -s tdd_quality_gate`
|
||||
- **Benchmarks:** `nox -s benchmark`
|
||||
- **Coverage report:** `nox -s coverage_report`
|
||||
|
||||
@@ -1275,7 +1274,6 @@ The CI pipeline runs the following jobs. Jobs without explicit dependencies run
|
||||
| `quality` | — | Radon complexity analysis via `nox -s complexity` |
|
||||
| `unit_tests` | — | Behave BDD tests via `nox -s unit_tests` |
|
||||
| `integration_tests` | — | Robot Framework integration tests via `nox -s integration_tests` |
|
||||
| `tdd_quality_gate` | — (PR-only) | TDD bug-fix workflow enforcement via `nox -s tdd_quality_gate` (checks bug refs, TDD tags, and expected-fail tag removal in PR diff) |
|
||||
| `e2e_tests` | — | End-to-end Robot tests with real LLM keys via `nox -s e2e_tests` |
|
||||
| `coverage` | `lint`, `typecheck` | Slipcover coverage report via `nox -s coverage_report` (fail-under 97%) |
|
||||
| `benchmark-regression` | `lint`, `typecheck` | ASV benchmark regression on PRs via `nox -s benchmark_regression` |
|
||||
@@ -1295,7 +1293,6 @@ mergeable:
|
||||
- **security** — No high-severity Bandit findings, Semgrep rules pass, no dead code
|
||||
- **unit_tests** — All Behave BDD scenarios pass
|
||||
- **coverage** — Test coverage >= 97%
|
||||
- **tdd_quality_gate** — On PRs only, bug-fix PRs must satisfy the TDD Bug Fix Workflow gate
|
||||
|
||||
#### How to Read CI Results
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* HAL 9000 <hal9000@cleverthis.com>
|
||||
* Hamza Khyari <hamza.khyari@cleverthis.com>
|
||||
* Jeffrey Phillips Freeman <jeffrey.freeman@syncleus.com>
|
||||
* Jeffrey Phillips Freeman <the@jeffreyfreeman.me>
|
||||
* Luis Mendes <luis.p.mendes@gmail.com>
|
||||
* Rui Hu <rui.hu@cleverthis.com>
|
||||
|
||||
@@ -13,15 +14,18 @@
|
||||
Below are some of the specific details of various contributions.
|
||||
|
||||
* Jeffrey Phillips Freeman has acted as Lead Developer, daily contributor, and Project Owner.
|
||||
* Jeffrey Phillips Freeman has contributed an implementation for the invariant propagation fix (PR #10881 / issue #9131): added `_propagate_invariant_decisions()` to `SubplanService` to propagate all `invariant_enforced` decisions from parent plans to child plan decision trees during subplan spawn, satisfying the specification requirement for invariant propagation across hierarchical plan execution.
|
||||
* Brent E. Edwards has contributed quality assurance, test coverage, and CI pipeline improvements.
|
||||
* HAL 9000 has contributed automated implementation, bug fixes, and feature development as part of the CleverAgents automation pool.
|
||||
* HAL 9000 has contributed concurrency safety improvements, including thread-safe context tier management (issue #7547) for parallel plan execution.
|
||||
* HAL 9000 has contributed the plan concurrency race-condition fix (#7989): wired `LockService` into the plan lifecycle, guarding `execute_plan()` and `apply_plan()` with plan-level advisory locks and unique per-invocation owner identities to prevent silent concurrent state corruption.
|
||||
<<<<<<< HEAD
|
||||
* HAL 9000 has contributed the bug-hunt-pool-supervisor non-blocking tracking fix (#7875 / PR #7957): updated step 5 to be best-effort and added rule 9 to prevent the automation-tracking-manager call from blocking the main supervisor loop.
|
||||
* Jeffrey Phillips Freeman has contributed the complete AUTO-BUG-POOL to AUTO-BUG-SUP tracking prefix fix across agent-system-specification.md, automation-tracking.md documentation and agent-system-specification.md spec document, replaced with correct `AUTO-BUG-SUP` prefix used by the bug-hunt-pool-supervisor agent (#7875).
|
||||
* HAL 9000 has contributed the plugin entry point security hardening fix (#7476): enforced entry point allowlist validation before importing plugin modules to prevent malicious plugin loading.
|
||||
* HAL 9000 has contributed the benchmark workflow separation (#9040): moved the benchmark-regression job out of the default PR workflow into a dedicated scheduled workflow, reducing median PR CI turnaround time from 99-132 minutes to under 30 minutes.
|
||||
* HAL 9000 has contributed the plan tree JSON/YAML command envelope fix (#9163): wrapped `agents plan tree --format json/yaml` output in the spec-required command envelope structure, added summary statistics, decision_ids mapping, child_plans list, and accurate timing measurement.
|
||||
* HAL 9000 contributed Structural Component Output Validation (PR #11147): implemented `validate_plan_tree`, `validate_decision_dict`, `validate_structured_output`, and `validate_structured_component_output` validators that replace exact-character matching with structural schema checking for plan tree nodes, decision CLI dictionaries, and structured session output envelopes.
|
||||
* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc.
|
||||
* HAL 9000 has contributed automated bug fixes, CLI output formatting improvements, and ongoing maintenance as part of the CleverAgents automation system.
|
||||
* HAL 9000 has contributed the file edit encoding parameter fix (PR #8258 / issue #7559).
|
||||
|
||||
@@ -21,6 +21,13 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
Then the actor should be registered with provider "anthropic" and model "claude-3"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
@tdd_issue @tdd_issue_4321 @tdd_issue_4300
|
||||
Scenario: registry.add() with nested-only type extracts provider/model from nested config
|
||||
When I add a YAML with type only in nested actors map and provider/model in nested config
|
||||
Then the actor should be registered with provider "custom" and model "fpga-model"
|
||||
And the registered actor name should be "local/fpga-strategist"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── actors: map with unsafe flag ───────────────────────────────────
|
||||
|
||||
Scenario: registry.add() preserves unsafe flag from nested spec-compliant config
|
||||
|
||||
+5
-15
@@ -699,21 +699,11 @@ def after_scenario(context, scenario):
|
||||
pass # Ignore cleanup errors
|
||||
context.test_dir = None
|
||||
|
||||
# Clean up temp directories created by tdd_quality_gate and other steps.
|
||||
# temp_dir may be a Path, str, or tempfile.TemporaryDirectory depending
|
||||
# on which step file set it; handle all three safely.
|
||||
# NOTE: do NOT set context.temp_dir = None here — cleanup functions
|
||||
# registered via context.add_cleanup() run AFTER after_scenario and may
|
||||
# still reference context.temp_dir (e.g. cli_init_yes_flag_steps.py).
|
||||
# The scenario context layer is popped after all cleanups finish, which
|
||||
# removes the attribute automatically.
|
||||
temp_dir = getattr(context, "temp_dir", None)
|
||||
if temp_dir is not None:
|
||||
if isinstance(temp_dir, (str, Path)):
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
elif hasattr(temp_dir, "cleanup"):
|
||||
with contextlib.suppress(Exception):
|
||||
temp_dir.cleanup()
|
||||
# Clean up TemporaryDirectory objects created by ACMS index traversal tests
|
||||
if hasattr(context, "temp_dir") and context.temp_dir is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
context.temp_dir.cleanup()
|
||||
context.temp_dir = None
|
||||
|
||||
# Clean up environment variables set during tests
|
||||
if hasattr(context, "env_vars_to_clean"):
|
||||
|
||||
@@ -69,9 +69,20 @@ Feature: Plan Generation LangGraph Coverage
|
||||
|
||||
Scenario: Should retry returns end when max retries reached
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 4
|
||||
Then the langgraph retry decision should be "end"
|
||||
|
||||
Scenario: Should retry still retries when retry_count equals max_retries
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 3
|
||||
Then the langgraph retry decision should be "retry"
|
||||
|
||||
Scenario: Validate node increments retry_count in returned state
|
||||
Given I have a langgraph PlanGenerationGraph instance
|
||||
When I execute the langgraph validate node with no changes
|
||||
Then the langgraph validation status should be "FAIL"
|
||||
And the langgraph validate result should include retry_count incremented from 0 to 1
|
||||
|
||||
Scenario: Validate node fails when no changes provided
|
||||
Given I have a langgraph PlanGenerationGraph instance
|
||||
When I execute the langgraph validate node with no changes
|
||||
|
||||
@@ -78,12 +78,13 @@ Feature: Plan Generation Graph Uncovered Lines Coverage
|
||||
And the langgraph async result should have generated_changes
|
||||
And the langgraph async result should have validation_result
|
||||
|
||||
# Testing line 403: Retry count increment in should_retry
|
||||
Scenario: Should retry increments retry count correctly
|
||||
# _should_retry is a read-only conditional-edge function; retry_count is
|
||||
# incremented inside _validate (a node) so LangGraph persists the new value.
|
||||
Scenario: Should retry does not mutate state and returns retry when retries remain
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
And I have a langgraph state with retry_count 1
|
||||
When I check uncovered langgraph should_retry with FAIL validation and retry_count 1
|
||||
Then the uncovered langgraph state retry_count should be incremented to 2
|
||||
Then the uncovered langgraph state retry_count should not be mutated and remain 1
|
||||
And the langgraph retry decision should be "retry"
|
||||
|
||||
# Testing full workflow with retry logic
|
||||
|
||||
@@ -143,6 +143,23 @@ def step_add_actors_separate(context: Context) -> None:
|
||||
context.spec_result = context.spec_registry.add(yaml_text)
|
||||
|
||||
|
||||
@when(
|
||||
"I add a YAML with type only in nested actors map and provider/model in nested config"
|
||||
)
|
||||
def step_add_nested_type_only(context: Context) -> None:
|
||||
yaml_text = (
|
||||
"name: local/fpga-strategist\n"
|
||||
"description: An FPGA strategist agent\n"
|
||||
"actors:\n"
|
||||
" fpga-strategist:\n"
|
||||
" type: llm\n"
|
||||
" config:\n"
|
||||
" provider: custom\n"
|
||||
" model: fpga-model\n"
|
||||
)
|
||||
context.spec_result = context.spec_registry.add(yaml_text)
|
||||
|
||||
|
||||
@when("I add a spec-compliant YAML with actors map and unsafe flag")
|
||||
def step_add_actors_unsafe(context: Context) -> None:
|
||||
yaml_text = (
|
||||
|
||||
@@ -252,14 +252,18 @@ def step_langgraph_graph_with_max_retries(context: Any, retries: int) -> None:
|
||||
"I check langgraph should_retry with {status} validation and retry_count {count:d}"
|
||||
)
|
||||
def step_check_langgraph_should_retry(context: Any, status: str, count: int) -> None:
|
||||
"""Check should_retry decision."""
|
||||
"""Check should_retry decision.
|
||||
|
||||
Note: _should_retry is a conditional-edge function and is read-only from
|
||||
LangGraph's perspective. The state dict passed here is NOT mutated by
|
||||
the function (retry_count is incremented inside _validate instead).
|
||||
"""
|
||||
state: dict[str, Any] = {
|
||||
"validation_result": {"status": status},
|
||||
"retry_count": count,
|
||||
}
|
||||
decision = context.graph._should_retry(state)
|
||||
context.retry_decision = decision
|
||||
context.final_retry_count = state.get("retry_count", count)
|
||||
|
||||
|
||||
@then('the langgraph retry decision should be "{decision}"')
|
||||
@@ -283,6 +287,28 @@ def step_langgraph_validation_status(context: Any, status: str) -> None:
|
||||
assert validation.get("status") == status
|
||||
|
||||
|
||||
@then(
|
||||
"the langgraph validate result should include retry_count incremented from {before:d} to {after:d}"
|
||||
)
|
||||
def step_langgraph_validate_result_retry_count(
|
||||
context: Any, before: int, after: int
|
||||
) -> None:
|
||||
"""Verify _validate returns an incremented retry_count.
|
||||
|
||||
The _validate node is responsible for incrementing retry_count so that
|
||||
LangGraph persists the new value into the graph state. Conditional-edge
|
||||
functions such as _should_retry are read-only from LangGraph's perspective
|
||||
and must not mutate state.
|
||||
"""
|
||||
result = context.node_result
|
||||
assert "retry_count" in result, (
|
||||
"_validate must return retry_count in its result dict"
|
||||
)
|
||||
assert result["retry_count"] == after, (
|
||||
f"Expected retry_count {after} (incremented from {before}), got {result['retry_count']}"
|
||||
)
|
||||
|
||||
|
||||
@then('the langgraph validation message should contain "{text}"')
|
||||
def step_langgraph_validation_message_contains(context: Any, text: str) -> None:
|
||||
"""Verify validation message contains text."""
|
||||
|
||||
@@ -459,16 +459,32 @@ def step_have_state_with_retry_count(context: Any, count: int) -> None:
|
||||
"I check uncovered langgraph should_retry with FAIL validation and retry_count {count:d}"
|
||||
)
|
||||
def step_check_should_retry_uncovered(context: Any, count: int) -> None:
|
||||
"""Check should_retry and verify retry_count increment."""
|
||||
"""Check should_retry returns correct decision.
|
||||
|
||||
_should_retry is a conditional-edge function — LangGraph treats it as
|
||||
read-only. The retry counter is already incremented inside _validate
|
||||
(a node whose return dict IS merged into the state), so _handle_retry
|
||||
is a simple pass-through bridge node.
|
||||
"""
|
||||
decision = context.graph._should_retry(context.state)
|
||||
context.retry_decision = decision
|
||||
context.final_retry_count = context.state.get("retry_count")
|
||||
|
||||
|
||||
@then("the uncovered langgraph state retry_count should be incremented to {expected:d}")
|
||||
def step_uncovered_retry_count_incremented(context: Any, expected: int) -> None:
|
||||
"""Verify retry_count was incremented."""
|
||||
assert context.final_retry_count == expected
|
||||
@then(
|
||||
"the uncovered langgraph state retry_count should not be mutated and remain {expected:d}"
|
||||
)
|
||||
def step_uncovered_retry_count_not_mutated(context: Any, expected: int) -> None:
|
||||
"""Verify _should_retry did not mutate retry_count in state.
|
||||
|
||||
_should_retry is a conditional-edge function; LangGraph treats it as
|
||||
read-only. The retry counter is incremented inside _validate (a node)
|
||||
so that LangGraph persists the new value into the graph state.
|
||||
"""
|
||||
assert context.final_retry_count == expected, (
|
||||
f"_should_retry must not mutate state['retry_count']: "
|
||||
f"expected {expected}, got {context.final_retry_count}"
|
||||
)
|
||||
|
||||
|
||||
# Workflow retry scenario
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Step definitions for structural component output validation.
|
||||
|
||||
Tests for features/structural_validation.feature - validates all four
|
||||
validators covering plan tree nodes, decision CLI dicts, structured
|
||||
output envelopes, and the unified dispatcher.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when # type: ignore[import-untyped]
|
||||
from behave.runner import Context # type: ignore[import-untyped]
|
||||
|
||||
from cleveragents.core.validation import (
|
||||
ValidationError,
|
||||
validate_decision_dict,
|
||||
validate_plan_tree,
|
||||
validate_structured_component_output,
|
||||
validate_structured_output,
|
||||
)
|
||||
|
||||
# Helper: valid ULID strings for test data
|
||||
VALID_ULID_A = "01ARZ3NDEKTSV4XXFFJFRC889A"
|
||||
VALID_ULID_B = "01ARZ3NDEKTSV4XXFFJFRC889B"
|
||||
VALID_ULID_C = "01ARZ3NDEKTSV4XXFFJFRC889C"
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Plan tree scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('the plan tree contains {node_count:d} node(s)')
|
||||
def step_plan_tree_node_count(ctx: Context, node_count: int) -> None:
|
||||
"""Create a list of valid plan tree nodes."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}",
|
||||
"type": "strategy_choice",
|
||||
"sequence": i,
|
||||
"question": f"Decision question #{i}",
|
||||
"children": [],
|
||||
**(
|
||||
{"parent_decision_id": VALID_ULID_A}
|
||||
if i > 0 else {}
|
||||
),
|
||||
}
|
||||
for i in range(node_count)
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node missing "decision_id"')
|
||||
def step_node_missing_decision_id(ctx: Context) -> None:
|
||||
"""Create a node missing the required decision_id field."""
|
||||
ctx.tree_nodes = [
|
||||
{"type": "strategy_choice", "sequence": 0, "question": "Missing ID", "children": []}
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node with invalid ULID "not-a-ulid-12345"')
|
||||
def step_node_invalid_ulid(ctx: Context) -> None:
|
||||
"""Create a node with an invalid decision_id."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": "not-a-ulid-12345",
|
||||
"type": "strategy_choice",
|
||||
"sequence": 0,
|
||||
"question": "Invalid ULID test",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node with empty "children" list')
|
||||
def step_node_empty_children(ctx: Context) -> None:
|
||||
"""Create a node with an empty children list - should be valid."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 0,
|
||||
"question": "Question text",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@given('two sibling nodes share the same sequence number')
|
||||
def step_duplicate_sequences(ctx: Context) -> None:
|
||||
"""Create two sibling nodes with identical sequence."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 5,
|
||||
"question": "First",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_B,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_C,
|
||||
"type": "implementation_choice",
|
||||
"sequence": 5,
|
||||
"question": "Second",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_B,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@given('a plan tree node with sequence "{seq}"')
|
||||
def step_node_sequence(ctx: Context, seq: str) -> None:
|
||||
"""Create a node with the given sequence value (as string to parse)."""
|
||||
int_seq = int(seq)
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": int_seq,
|
||||
"question": "Q",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Decision dict scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_value(raw: str) -> Any:
|
||||
"""Parse a behave value string into Python type."""
|
||||
stripped = raw.strip().strip('"').strip("'")
|
||||
if stripped in ("null", "none", "None"):
|
||||
return None
|
||||
if stripped == "true":
|
||||
return True
|
||||
if stripped == "false":
|
||||
return False
|
||||
try:
|
||||
return int(stripped)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(stripped)
|
||||
except ValueError:
|
||||
pass
|
||||
return stripped
|
||||
|
||||
|
||||
def _ensure_decision_dict(ctx: Context) -> dict:
|
||||
"""Ensure ctx.decision_dict exists with defaults."""
|
||||
if not hasattr(ctx, "decision_dict"):
|
||||
ctx.decision_dict = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Which approach?",
|
||||
"chosen": "Option A",
|
||||
"confidence": 0.75,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
return ctx.decision_dict # type: ignore[return-value]
|
||||
|
||||
|
||||
@given('I have a decision dict with "{field}" set to {value}')
|
||||
def step_decision_dict_field(ctx: Context, field: str, value) -> Any: # type: ignore[no-untyped-def]
|
||||
"""Set a single field on the decision dict."""
|
||||
d = _ensure_decision_dict(ctx)
|
||||
d[field] = _parse_value(str(value))
|
||||
|
||||
|
||||
@given('I have a valid decision dict')
|
||||
@when('I validate the decision dict')
|
||||
def step_validate_decision(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
d = _ensure_decision_dict(ctx)
|
||||
ctx.decision_result = validate_decision_dict(d)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Structured output scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
def _ensure_struct_output(ctx: Context) -> dict:
|
||||
"""Ensure ctx.struct_output exists with defaults."""
|
||||
if not hasattr(ctx, "struct_output"):
|
||||
ctx.struct_output = {
|
||||
"command": "agents plan list",
|
||||
"session_id": VALID_ULID_A,
|
||||
"status": "ok",
|
||||
"exit_code": 0,
|
||||
"elements": [],
|
||||
}
|
||||
return ctx.struct_output # type: ignore[return-value]
|
||||
|
||||
|
||||
@given('I have a valid structured output with all required fields')
|
||||
@when('I validate the structured output')
|
||||
def step_validate_structured(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
ctx.struct_result = validate_structured_output(o)
|
||||
|
||||
|
||||
@given('I have a structured output with "{field}" set to {value} (and all other required fields present)')
|
||||
def step_structured_output_field(ctx: Context, field: str, value) -> Any: # type: ignore[no-untyped-def]
|
||||
"""Set a single field on the structured output."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o[field] = _parse_value(str(value))
|
||||
|
||||
|
||||
@given('I have a structured output with empty "elements" list (and all other fields valid)')
|
||||
def step_empty_elements(ctx: Context) -> None:
|
||||
"""Empty elements is valid."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o["elements"] = []
|
||||
|
||||
|
||||
@given('I have a structured output with an element missing "kind" field')
|
||||
def step_invalid_element(ctx: Context) -> None:
|
||||
"""Create an invalid element."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o["elements"] = [{"not_kind": "panel"}]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Dispatcher scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('I target validation for "{target}" with valid data')
|
||||
def step_dispatcher_target(ctx: Context, target: str) -> None:
|
||||
"""Set up a dispatcher test."""
|
||||
ctx.target_type = target
|
||||
if target in ("plan_tree", "plan-tree", "decisions", "decision-tree", "tree"):
|
||||
ctx.validator_data = [
|
||||
{"decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 0, "question": "Q", "children": []}
|
||||
]
|
||||
elif target in ("decision", "decision_cli", "decision-cli", "cli_dict", "as_cli_dict"):
|
||||
ctx.validator_data = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Q?",
|
||||
"chosen": "A",
|
||||
"confidence": 0.5,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
elif target in ("structured_output", "structured-output", "session_output", "output_session"):
|
||||
ctx.validator_data = {
|
||||
"command": "test",
|
||||
"session_id": VALID_ULID_A,
|
||||
"status": "ok",
|
||||
"exit_code": 0,
|
||||
"elements": [],
|
||||
}
|
||||
else:
|
||||
ctx.validator_data = {}
|
||||
|
||||
|
||||
@when('I call validate_structured_component_output')
|
||||
def step_call_dispatcher(ctx: Context) -> None:
|
||||
"""Call the dispatcher."""
|
||||
target = ctx.target_type if hasattr(ctx, "target_type") else "plan_tree" # type: ignore[attr-defined]
|
||||
data = ctx.validator_data if hasattr(ctx, "validator_data") else {} # type: ignore[attr-defined]
|
||||
try:
|
||||
ctx.dispatcher_result = validate_structured_component_output(target, data) # type: ignore[arg-type]
|
||||
ctx.dispatcher_error = None
|
||||
except ValidationError as exc:
|
||||
ctx.dispatcher_result = None
|
||||
ctx.dispatcher_error = exc
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Shared scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('the plan tree has {node_count:d} valid nodes')
|
||||
def step_valid_tree(ctx: Context, node_count: int = 3) -> None:
|
||||
"""Set up a default valid tree."""
|
||||
if not hasattr(ctx, "tree_nodes"):
|
||||
ctx.tree_nodes = [
|
||||
{"decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}",
|
||||
"type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": []}
|
||||
for i in range(node_count)
|
||||
]
|
||||
|
||||
|
||||
@given('I have a plan tree with valid nodes')
|
||||
def step_valid_nodes_default(ctx: Context) -> None:
|
||||
"""Set default valid tree if no tree defined."""
|
||||
_valid_tree = False # handled by node_count scenario; nothing extra
|
||||
|
||||
|
||||
@when('I validate the plan tree')
|
||||
def step_validate_plan_tree(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
nodes = ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [ # type: ignore[attr-defined]
|
||||
{"decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": []}
|
||||
for i in range(3)
|
||||
]
|
||||
ctx.validation_result = validate_plan_tree(nodes)
|
||||
|
||||
|
||||
@when('I validate the plan tree structure')
|
||||
def step_validate_structure(ctx: Context) -> None:
|
||||
"""Run plan_tree validation."""
|
||||
nodes = ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [] # type: ignore[attr-defined]
|
||||
ctx.validation_result = validate_plan_tree(nodes)
|
||||
|
||||
|
||||
@given('I have a decision dict where whitespace may vary between fields')
|
||||
def step_whitespace_dict(ctx: Context) -> None:
|
||||
"""Whitespace in string values - structure still valid."""
|
||||
ctx.decision_dict = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": " strategy_choice ",
|
||||
"sequence": 1,
|
||||
"question": "Which approach? ",
|
||||
"chosen": " Option A ",
|
||||
"confidence": 0.75,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
|
||||
|
||||
@given('a plan tree with parent and child node references connected by ULID')
|
||||
def step_parent_child_tree(ctx: Context) -> None:
|
||||
"""Parent-decoy pattern - root plus children."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "prompt_definition",
|
||||
"sequence": 0,
|
||||
"question": "Root question",
|
||||
"children": [{"decision_id": VALID_ULID_B}, {"decision_id": VALID_ULID_C}],
|
||||
"parent_decision_id": None,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Child A",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_A,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_C,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 2,
|
||||
"question": "Child B",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_A,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@given('I have invalid plan tree data')
|
||||
def step_invalid_tree_data(ctx: Context) -> None:
|
||||
"""Create invalid plan tree node."""
|
||||
ctx.tree_nodes = [
|
||||
{"not_a_key": "value"}, # missing required fields entirely
|
||||
]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Assertions (shared / cross-scenario)
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@then('it should be structurally valid')
|
||||
def step_should_be_valid(ctx: Context) -> None:
|
||||
"""Assert validation passed."""
|
||||
result = getattr(ctx, "validation_result", {}) or {"valid": True}
|
||||
assert result["valid"], f"Expected valid but got errors: {result.get('errors', [])}"
|
||||
|
||||
|
||||
@then('it should be structurally invalid')
|
||||
def step_should_be_invalid(ctx: Context) -> None:
|
||||
"""Assert validation failed."""
|
||||
result = getattr(ctx, "validation_result", {}) or {"valid": True}
|
||||
assert not result["valid"], f"Expected invalid but got valid"
|
||||
|
||||
|
||||
@then('it should report no errors')
|
||||
def step_no_errors(ctx: Context) -> None:
|
||||
"""Assert no error messages."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert len(errs) == 0, f"Expected no errors but got: {errs}"
|
||||
|
||||
|
||||
@then('it should report errors including "{text}"')
|
||||
def step_errors_include(ctx: Context, text: str) -> None:
|
||||
"""Assert at least one error contains the given substring."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}"
|
||||
|
||||
|
||||
@then('error message must include "{text}"')
|
||||
@then('error must include "{text}"')
|
||||
def step_error_contains(ctx: Context, text: str) -> None:
|
||||
"""Assert an error contains substring."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}"
|
||||
|
||||
|
||||
@then('the first error message should identify "{prefix}" and the specific field issue')
|
||||
def step_first_error_identifies(ctx: Context, prefix: str) -> None:
|
||||
"""Assert first error contains prefix."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert len(errs) > 0, "Expected at least one error"
|
||||
assert prefix in errs[0], f"First error '{errs[0]}' should contain '{prefix}'"
|
||||
|
||||
|
||||
@then('it should raise a ValidationError with message containing "{text}"')
|
||||
def step_raise_error_with_text(ctx: Context, text: str) -> None:
|
||||
"""Assert dispatcher raised specific error."""
|
||||
err = getattr(ctx, "dispatcher_error", None)
|
||||
assert err is not None, "Expected a ValidationError"
|
||||
assert isinstance(err, ValidationError), f"Expected ValidationError, got {type(err)}"
|
||||
assert text in str(err), f"'{text}' not in error message: {err}"
|
||||
|
||||
|
||||
@then('it should dispatch to the matching validator')
|
||||
@then('return a valid result')
|
||||
def step_dispatcher_valid(ctx: Context) -> None:
|
||||
"""Assert dispatcher succeeded and returned valid."""
|
||||
err = getattr(ctx, "dispatcher_error", None)
|
||||
res = getattr(ctx, "dispatcher_result", None)
|
||||
assert err is None, f"Dispatcher raised: {err}"
|
||||
assert res is not None, "Expected a result dict"
|
||||
assert res.get("valid"), f"Result not valid: {res}"
|
||||
|
||||
|
||||
@then('validation should pass (structure matched, not exact characters)')
|
||||
def step_structure_matched(ctx: Context) -> None:
|
||||
"""Whitespace-tolerant structural validation passes."""
|
||||
d = getattr(ctx, "decision_dict", {})
|
||||
ctx.validation_result = validate_decision_dict(d) # type: ignore[attr-defined]
|
||||
assert ctx.validation_result["valid"], f"Structural validation failed: {ctx.validation_result.get('errors', [])}" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then('parent-child relationships should be validly connected')
|
||||
def step_relationships_valid(ctx: Context) -> None:
|
||||
"""Parent-child nodes validate without errors."""
|
||||
result = ctx.validation_result if hasattr(ctx, "validation_result") else {} # type: ignore[attr-defined]
|
||||
assert result.get("valid", False), f"Relation validation failed: {result.get('errors', [])}"
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Steps for tdd_cleanup_stale_destroys_execute_output.feature.
|
||||
|
||||
TDD issue-capture test for bug #11121:
|
||||
_create_sandbox_for_plan() calls GitWorktreeSandbox.cleanup_stale() unconditionally,
|
||||
destroying the cleveragents/plan-<id> branch when the plan is already in
|
||||
execute/complete or execute/processing state (awaiting apply or still in progress).
|
||||
|
||||
The scenarios are tagged @tdd_issue_11121 as permanent regression guards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
_PLAN_ID = "01TDDSANDBOX000000000000A"
|
||||
_BRANCH_NAME = f"cleveragents/plan-{_PLAN_ID}"
|
||||
|
||||
|
||||
def _git(args: list[str], cwd: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
def _branch_exists(repo_path: str, branch: str) -> bool:
|
||||
"""Return True if the given branch exists in the repo."""
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "--verify", f"refs/heads/{branch}"],
|
||||
cwd=repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _init_git_repo(path: str) -> None:
|
||||
_git(["init", "-q", "-b", "main"], path)
|
||||
_git(["config", "user.name", "TDD Test"], path)
|
||||
_git(["config", "user.email", "tdd@test.local"], path)
|
||||
_git(["config", "commit.gpgsign", "false"], path)
|
||||
|
||||
|
||||
def _build_execute_complete_mocks(
|
||||
context: Context,
|
||||
repo_path: str,
|
||||
) -> None:
|
||||
"""Build mock service + container for a plan in execute/complete state."""
|
||||
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
|
||||
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_type_name = "git-checkout"
|
||||
mock_resource.location = repo_path
|
||||
mock_resource.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_lr = MagicMock()
|
||||
mock_lr.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_project.linked_resources = [mock_lr]
|
||||
|
||||
# Plan is in execute/complete state — this is the critical state for the bug
|
||||
mock_plan = MagicMock()
|
||||
mock_plan.project_links = [MagicMock(project_name="local/tdd-11121-project")]
|
||||
mock_plan.phase = PlanPhase.EXECUTE
|
||||
mock_plan.processing_state = ProcessingState.COMPLETE
|
||||
mock_plan.state = ProcessingState.COMPLETE
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_plan.return_value = mock_plan
|
||||
|
||||
mock_project_repo = MagicMock()
|
||||
mock_project_repo.get.return_value = mock_project
|
||||
|
||||
mock_resource_registry = MagicMock()
|
||||
mock_resource_registry.show_resource.return_value = mock_resource
|
||||
|
||||
mock_container = MagicMock()
|
||||
mock_container.namespaced_project_repo.return_value = mock_project_repo
|
||||
mock_container.resource_registry_service.return_value = mock_resource_registry
|
||||
|
||||
context.tdd11121_service = mock_service
|
||||
context.tdd11121_container = mock_container
|
||||
context.tdd11121_repo_path = repo_path
|
||||
context.tdd11121_plan_id = _PLAN_ID
|
||||
context.tdd11121_branch = _BRANCH_NAME
|
||||
|
||||
|
||||
@given("a temp git repo with an execute-output branch for tdd 11121")
|
||||
def step_create_git_repo_with_execute_output(context: Context) -> None:
|
||||
"""Create a real git repo with a cleveragents/plan-<id> branch holding execute output.
|
||||
|
||||
This simulates the state after a successful plan execute: the worktree branch
|
||||
exists and contains at least one committed file representing execution output.
|
||||
"""
|
||||
d = tempfile.mkdtemp(prefix="tdd-11121-")
|
||||
context.add_cleanup(shutil.rmtree, d, True)
|
||||
|
||||
# Initialise the repo with a base commit on main
|
||||
_init_git_repo(d)
|
||||
Path(d, "README.md").write_text("# project\n")
|
||||
_git(["add", "."], d)
|
||||
_git(["commit", "-q", "-m", "init: base commit"], d)
|
||||
|
||||
# Create the cleveragents/plan-<id> branch and commit an output file to it.
|
||||
# This simulates what _commit_worktree_changes() does after execute completes.
|
||||
_git(["checkout", "-q", "-b", _BRANCH_NAME], d)
|
||||
output_file = Path(d, "generated_output.py")
|
||||
output_file.write_text("# Generated by plan execute\nresult = 42\n")
|
||||
_git(["add", "."], d)
|
||||
_git(["commit", "-q", "-m", f"cleveragents: execute output for plan {_PLAN_ID}"], d)
|
||||
|
||||
# Return to main so the repo is in a normal state
|
||||
_git(["checkout", "-q", "main"], d)
|
||||
|
||||
context.tdd11121_repo_path = d
|
||||
# Verify the branch exists before the test
|
||||
assert _branch_exists(d, _BRANCH_NAME), (
|
||||
f"Pre-condition failed: branch {_BRANCH_NAME} should exist before test"
|
||||
)
|
||||
|
||||
|
||||
def _build_execute_processing_mocks(
|
||||
context: Context,
|
||||
repo_path: str,
|
||||
) -> None:
|
||||
"""Build mock service + container for a plan in execute/processing state."""
|
||||
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
|
||||
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_type_name = "git-checkout"
|
||||
mock_resource.location = repo_path
|
||||
mock_resource.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_lr = MagicMock()
|
||||
mock_lr.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_project.linked_resources = [mock_lr]
|
||||
|
||||
mock_plan = MagicMock()
|
||||
mock_plan.project_links = [MagicMock(project_name="local/tdd-11121-project")]
|
||||
mock_plan.phase = PlanPhase.EXECUTE
|
||||
mock_plan.processing_state = ProcessingState.PROCESSING
|
||||
mock_plan.state = ProcessingState.PROCESSING
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_plan.return_value = mock_plan
|
||||
|
||||
mock_project_repo = MagicMock()
|
||||
mock_project_repo.get.return_value = mock_project
|
||||
|
||||
mock_resource_registry = MagicMock()
|
||||
mock_resource_registry.show_resource.return_value = mock_resource
|
||||
|
||||
mock_container = MagicMock()
|
||||
mock_container.namespaced_project_repo.return_value = mock_project_repo
|
||||
mock_container.resource_registry_service.return_value = mock_resource_registry
|
||||
|
||||
context.tdd11121_service = mock_service
|
||||
context.tdd11121_container = mock_container
|
||||
context.tdd11121_repo_path = repo_path
|
||||
context.tdd11121_plan_id = _PLAN_ID
|
||||
context.tdd11121_branch = _BRANCH_NAME
|
||||
|
||||
|
||||
@given("a mocked plan service with the plan in execute/complete state for tdd 11121")
|
||||
def step_mock_service_execute_complete(context: Context) -> None:
|
||||
"""Set up mock service returning a plan in execute/complete state."""
|
||||
_build_execute_complete_mocks(context, context.tdd11121_repo_path)
|
||||
|
||||
|
||||
@given("a mocked plan service with the plan in execute/processing state for tdd 11121")
|
||||
def step_mock_service_execute_processing(context: Context) -> None:
|
||||
"""Set up mock service returning a plan in execute/processing state."""
|
||||
_build_execute_processing_mocks(context, context.tdd11121_repo_path)
|
||||
|
||||
|
||||
@when(
|
||||
"I call _create_sandbox_for_plan a second time on the execute/complete plan for tdd 11121"
|
||||
)
|
||||
def step_call_create_sandbox_second_time(context: Context) -> None:
|
||||
"""Call _create_sandbox_for_plan on a plan already in execute/complete state.
|
||||
|
||||
This simulates the user re-running 'agents plan execute <PLAN_ID>' after
|
||||
execution has already completed. The bug causes cleanup_stale to run and
|
||||
destroy the cleveragents/plan-<id> branch that holds the execute output.
|
||||
"""
|
||||
from cleveragents.cli.commands.plan import _create_sandbox_for_plan
|
||||
|
||||
context.tdd11121_second_call_exception: Exception | None = None
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.application.container.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
):
|
||||
_sandbox_root, sandbox_infos = _create_sandbox_for_plan(
|
||||
context.tdd11121_plan_id,
|
||||
context.tdd11121_service,
|
||||
)
|
||||
context.tdd11121_sandbox_infos = sandbox_infos
|
||||
# Clean up any newly created sandbox to avoid resource leaks
|
||||
for sinfo in sandbox_infos:
|
||||
with contextlib.suppress(Exception):
|
||||
sinfo.sandbox_obj.cleanup()
|
||||
except Exception as exc:
|
||||
# Record but don't re-raise — we want to check branch state regardless
|
||||
context.tdd11121_second_call_exception = exc
|
||||
|
||||
|
||||
@when(
|
||||
"I call _create_sandbox_for_plan a second time on the execute/processing plan for tdd 11121"
|
||||
)
|
||||
def step_call_create_sandbox_second_time_processing(context: Context) -> None:
|
||||
"""Call _create_sandbox_for_plan on a plan already in execute/processing state.
|
||||
|
||||
This simulates the user re-running 'agents plan execute <PLAN_ID>' while the
|
||||
plan is still in progress (execute/processing). The guard must protect the
|
||||
sandbox branch from cleanup_stale in this state too.
|
||||
"""
|
||||
from cleveragents.cli.commands.plan import _create_sandbox_for_plan
|
||||
|
||||
context.tdd11121_second_call_exception: Exception | None = None
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.application.container.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
):
|
||||
_sandbox_root, sandbox_infos = _create_sandbox_for_plan(
|
||||
context.tdd11121_plan_id,
|
||||
context.tdd11121_service,
|
||||
)
|
||||
context.tdd11121_sandbox_infos = sandbox_infos
|
||||
for sinfo in sandbox_infos:
|
||||
with contextlib.suppress(Exception):
|
||||
sinfo.sandbox_obj.cleanup()
|
||||
except Exception as exc:
|
||||
context.tdd11121_second_call_exception = exc
|
||||
|
||||
|
||||
@then(
|
||||
"the cleveragents plan branch should still exist after the second call for tdd 11121"
|
||||
)
|
||||
def step_branch_still_exists(context: Context) -> None:
|
||||
"""Assert the cleveragents/plan-<id> branch was NOT destroyed by cleanup_stale.
|
||||
|
||||
The guard in _create_sandbox_for_plan skips cleanup_stale when the plan is
|
||||
already in execute/processing or execute/complete state, preserving the
|
||||
sandbox branch so plan apply can merge it.
|
||||
"""
|
||||
repo_path = context.tdd11121_repo_path
|
||||
branch = context.tdd11121_branch
|
||||
assert _branch_exists(repo_path, branch), (
|
||||
f"Bug #11121: branch '{branch}' was destroyed by cleanup_stale during "
|
||||
f"a second _create_sandbox_for_plan call. "
|
||||
f"The branch must survive until plan apply merges it."
|
||||
)
|
||||
|
||||
|
||||
@then("the apply sandbox changes should find at least one artifact for tdd 11121")
|
||||
def step_apply_finds_artifacts(context: Context) -> None:
|
||||
"""Assert that apply can find the execute output after a re-invoked execute.
|
||||
|
||||
The guard in _create_sandbox_for_plan preserves the sandbox branch, so the
|
||||
git diff between HEAD and the plan branch finds at least one artifact.
|
||||
"""
|
||||
repo_path = context.tdd11121_repo_path
|
||||
branch = context.tdd11121_branch
|
||||
|
||||
if not _branch_exists(repo_path, branch):
|
||||
raise AssertionError(
|
||||
f"Bug #11121: branch '{branch}' was destroyed by cleanup_stale. "
|
||||
f"plan apply would find 0 artifacts (empty changeset). "
|
||||
f"The branch must persist until apply merges it."
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--stat", f"HEAD...{branch}"],
|
||||
cwd=repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
stat_lines = (result.stdout or "").strip().splitlines()
|
||||
artifact_count = max(0, len(stat_lines) - 1) if stat_lines else 0
|
||||
|
||||
assert artifact_count > 0, (
|
||||
f"Bug #11121: plan apply would find {artifact_count} artifacts after "
|
||||
f"re-invoked execute on execute/complete plan. Expected >= 1 artifact. "
|
||||
f"diff --stat output: {result.stdout!r}"
|
||||
)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Step definitions for TDD Issue #9131 — invariant_enforced decisions not
|
||||
propagated to child plans on subplan spawn.
|
||||
|
||||
These steps exercise ``SubplanService.spawn()`` and verify that it propagates
|
||||
``invariant_enforced`` decisions from the parent plan to each child plan's
|
||||
decision tree.
|
||||
|
||||
On ``master`` (before the fix), ``spawn()`` creates child Plan objects but
|
||||
does NOT record ``invariant_enforced`` decisions on them. The assertions in
|
||||
these steps will **fail** until the bug is fixed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.application.services.subplan_service import (
|
||||
SpawnEntry,
|
||||
SpawnResult,
|
||||
SubplanService,
|
||||
)
|
||||
from cleveragents.domain.models.core.decision import (
|
||||
ContextSnapshot,
|
||||
Decision,
|
||||
DecisionType,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
ExecutionMode,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
SubplanConfig,
|
||||
)
|
||||
|
||||
_PARENT_PLAN_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6PN00"
|
||||
_ROOT_PLAN_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6RF00"
|
||||
_SPAWN_DEC_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6DA00"
|
||||
|
||||
|
||||
def _make_parent_plan() -> Plan:
|
||||
"""Create a minimal parent plan for testing."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
root_plan_id=_ROOT_PLAN_ID,
|
||||
),
|
||||
namespaced_name=NamespacedName(namespace="local", name="parent-plan"),
|
||||
description="Parent plan for invariant propagation test",
|
||||
action_name="local/parent-action",
|
||||
)
|
||||
|
||||
|
||||
def _make_spawn_decision(
|
||||
plan_id: str = _PARENT_PLAN_ID,
|
||||
) -> Decision:
|
||||
"""Create a subplan_spawn Decision."""
|
||||
return Decision(
|
||||
decision_id=_SPAWN_DEC_ID,
|
||||
plan_id=plan_id,
|
||||
decision_type=DecisionType.SUBPLAN_SPAWN,
|
||||
sequence_number=10,
|
||||
question="Should we spawn a child plan?",
|
||||
chosen_option="local/sub-action",
|
||||
context_snapshot=ContextSnapshot(),
|
||||
)
|
||||
|
||||
|
||||
def _make_spawn_entry(plan_id: str = _PARENT_PLAN_ID) -> SpawnEntry:
|
||||
"""Create a single spawn entry."""
|
||||
return SpawnEntry(
|
||||
decision=_make_spawn_decision(plan_id),
|
||||
action_name="local/sub-action",
|
||||
description="Child plan for invariant propagation test",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a parent plan with invariant_enforced decisions recorded")
|
||||
def step_parent_with_invariant_decisions(context: Context) -> None:
|
||||
"""Set up a parent plan with one invariant_enforced decision."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
# Record one invariant_enforced decision on the parent plan
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question="Is this invariant enforced?",
|
||||
chosen_option="invariant enforced: do not modify production data",
|
||||
)
|
||||
context.parent_invariant_decisions = [inv_dec]
|
||||
context.expected_invariant_count = 1
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with {count:d} invariant_enforced decisions recorded")
|
||||
def step_parent_with_n_invariant_decisions(context: Context, count: int) -> None:
|
||||
"""Set up a parent plan with N invariant_enforced decisions."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
inv_decisions: list[Decision] = []
|
||||
for i in range(count):
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question=f"Is invariant {i} enforced?",
|
||||
chosen_option=f"invariant enforced: constraint {i}",
|
||||
)
|
||||
inv_decisions.append(inv_dec)
|
||||
|
||||
context.parent_invariant_decisions = inv_decisions
|
||||
context.expected_invariant_count = count
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with a non_overridable invariant_enforced decision")
|
||||
def step_parent_with_non_overridable_invariant(context: Context) -> None:
|
||||
"""Set up a parent plan with a non_overridable invariant_enforced decision."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
# Record a non_overridable invariant decision
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question="Is this non-overridable global invariant enforced?",
|
||||
chosen_option="non_overridable invariant enforced: never delete user data",
|
||||
rationale="Global non-overridable invariant must propagate to all child plans",
|
||||
)
|
||||
context.parent_invariant_decisions = [inv_dec]
|
||||
context.non_overridable_chosen_option = inv_dec.chosen_option
|
||||
context.expected_invariant_count = 1
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with no invariant_enforced decisions")
|
||||
def step_parent_with_no_invariant_decisions(context: Context) -> None:
|
||||
"""Set up a parent plan with no invariant_enforced decisions."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
context.parent_invariant_decisions = []
|
||||
context.expected_invariant_count = 0
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a valid spawn entry for the parent plan")
|
||||
def step_valid_spawn_entry(context: Context) -> None:
|
||||
"""Create a single valid spawn entry."""
|
||||
context.spawn_entries = [_make_spawn_entry()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I spawn a child plan via SubplanService")
|
||||
def step_spawn_child_plan(context: Context) -> None:
|
||||
"""Call SubplanService.spawn() and capture the result."""
|
||||
context.spawn_error = None
|
||||
try:
|
||||
context.spawn_result = context.subplan_service.spawn(
|
||||
parent_plan=context.parent_plan,
|
||||
config=context.subplan_config,
|
||||
spawn_entries=context.spawn_entries,
|
||||
)
|
||||
except Exception as exc:
|
||||
context.spawn_error = exc
|
||||
context.spawn_result = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then(
|
||||
"the decision service should have recorded invariant_enforced decisions"
|
||||
" for the child plan"
|
||||
)
|
||||
def step_child_has_invariant_decisions(context: Context) -> None:
|
||||
"""Assert that invariant_enforced decisions were recorded for the child plan.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions to
|
||||
child plans. This assertion will fail until the bug is fixed.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
# Query the decision service for invariant_enforced decisions on the child plan
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) > 0, (
|
||||
f"Child plan {child_plan_id!r} has no invariant_enforced decisions. "
|
||||
f"Bug #9131: SubplanService.spawn() does not propagate "
|
||||
f"invariant_enforced decisions from parent plan to child plans."
|
||||
)
|
||||
|
||||
|
||||
@then("the child plan should have {count:d} invariant_enforced decisions recorded")
|
||||
def step_child_has_n_invariant_decisions(context: Context, count: int) -> None:
|
||||
"""Assert that the child plan has exactly N invariant_enforced decisions.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) == count, (
|
||||
f"Child plan {child_plan_id!r} has {len(child_invariant_decisions)} "
|
||||
f"invariant_enforced decisions, expected {count}. "
|
||||
f"Bug #9131: SubplanService.spawn() does not propagate "
|
||||
f"invariant_enforced decisions from parent plan to child plans."
|
||||
)
|
||||
|
||||
|
||||
@then("the child plan should have the non_overridable invariant decision propagated")
|
||||
def step_child_has_non_overridable_invariant(context: Context) -> None:
|
||||
"""Assert that the non_overridable invariant decision was propagated.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) > 0, (
|
||||
f"Child plan {child_plan_id!r} has no invariant_enforced decisions. "
|
||||
f"Bug #9131: non_overridable invariant was not propagated."
|
||||
)
|
||||
|
||||
# Verify the non_overridable invariant text was propagated
|
||||
expected_option: str = context.non_overridable_chosen_option
|
||||
propagated_options: list[str] = [d.chosen_option for d in child_invariant_decisions]
|
||||
assert expected_option in propagated_options, (
|
||||
f"Non-overridable invariant chosen_option {expected_option!r} not found "
|
||||
f"in child plan decisions: {propagated_options}. "
|
||||
f"Bug #9131: non_overridable invariant was not propagated."
|
||||
)
|
||||
|
||||
|
||||
@then("the spawn should succeed with no invariant decisions recorded for the child")
|
||||
def step_spawn_succeeds_no_invariants(context: Context) -> None:
|
||||
"""Assert that spawn succeeds cleanly when parent has no invariant decisions."""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert result.total_spawned == 1, (
|
||||
f"Expected 1 spawned child, got {result.total_spawned}"
|
||||
)
|
||||
|
||||
# When parent has no invariant decisions, child should also have none
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) == 0, (
|
||||
f"Child plan {child_plan_id!r} has {len(child_invariant_decisions)} "
|
||||
f"invariant_enforced decisions, expected 0 (parent had none)."
|
||||
)
|
||||
@@ -1,541 +0,0 @@
|
||||
"""Step definitions for TDD quality gate feature tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
# Ensure the project root is importable.
|
||||
_ROOT = str(Path(__file__).resolve().parents[2])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from scripts.tdd_quality_gate import ( # noqa: E402
|
||||
check_expected_fail_removed,
|
||||
find_tdd_tests,
|
||||
parse_bug_refs,
|
||||
run_quality_gate,
|
||||
)
|
||||
from scripts.tdd_quality_gate import main as quality_gate_main # noqa: E402
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_temp_dir(context: object) -> Path:
|
||||
"""Return (and lazily create) the temporary directory on context."""
|
||||
tmp: Path | None = getattr(context, "temp_dir", None)
|
||||
if tmp is None:
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
context.temp_dir = tmp # type: ignore[attr-defined]
|
||||
return tmp
|
||||
|
||||
|
||||
def _cleanup_temp_dir(context: object) -> None:
|
||||
"""Remove the temporary directory if it exists."""
|
||||
tmp: Path | None = getattr(context, "temp_dir", None)
|
||||
if tmp is not None and tmp.exists():
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def _default_pr_diff_for_bug_refs(
|
||||
bug_refs: list[int], search_root: Path | None = None
|
||||
) -> str:
|
||||
"""Return a synthetic PR diff that removes expected-fail tags.
|
||||
|
||||
When *search_root* is provided the helper inspects the temp tree to
|
||||
decide whether each bug's TDD test lives in a ``.feature`` or
|
||||
``.robot`` file and emits the diff in the matching format.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
for bug_num in bug_refs:
|
||||
use_robot = False
|
||||
if search_root is not None:
|
||||
robot_hits = list(search_root.rglob("*.robot"))
|
||||
for rp in robot_hits:
|
||||
try:
|
||||
if f"tdd_bug_{bug_num}" in rp.read_text(encoding="utf-8"):
|
||||
use_robot = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if use_robot:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
f"diff --git a/robot/bug{bug_num}.robot b/robot/bug{bug_num}.robot",
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
else:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
f"diff --git a/features/bug{bug_num}.feature b/features/bug{bug_num}.feature",
|
||||
f"--- a/features/bug{bug_num}.feature",
|
||||
f"+++ b/features/bug{bug_num}.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}",
|
||||
f"+@tdd_bug @tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a PR description "{description}"')
|
||||
def step_given_pr_description(context: object, description: str) -> None:
|
||||
context.pr_description = description # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@given("a multiline PR description")
|
||||
def step_given_multiline_pr_description(context: object) -> None:
|
||||
context.pr_description = context.text # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I parse the bug references")
|
||||
def step_when_parse_bug_refs(context: object) -> None:
|
||||
context.bug_refs = parse_bug_refs( # type: ignore[attr-defined]
|
||||
context.pr_description # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then("the bug references should be [{refs}]")
|
||||
def step_then_bug_refs_should_be(context: object, refs: str) -> None:
|
||||
if refs.strip() == "":
|
||||
expected: list[int] = []
|
||||
else:
|
||||
expected = [int(x.strip()) for x in refs.split(",")]
|
||||
actual: list[int] = context.bug_refs # type: ignore[attr-defined]
|
||||
if actual != expected:
|
||||
raise AssertionError(f"Expected bug refs {expected}, got {actual}")
|
||||
|
||||
|
||||
@then("the bug references should be []")
|
||||
def step_then_bug_refs_empty(context: object) -> None:
|
||||
actual: list[int] = context.bug_refs # type: ignore[attr-defined]
|
||||
if actual != []:
|
||||
raise AssertionError(f"Expected empty bug refs, got {actual}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a temporary directory with a file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_dir_with_file(context: object, filepath: str, content: str) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given('a temporary directory also has a file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_dir_also_has_file(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@when("I search for TDD tests for bug {bug_num:d}")
|
||||
def step_when_search_tdd_tests(context: object, bug_num: int) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
context.found_tests = find_tdd_tests(bug_num, tmp) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("the search should find {count:d} test file")
|
||||
def step_then_search_finds_count(context: object, count: int) -> None:
|
||||
actual = len(context.found_tests) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(f"Expected {count} test file(s), found {actual}")
|
||||
|
||||
|
||||
@then("the search should find {count:d} test files")
|
||||
def step_then_search_finds_count_plural(context: object, count: int) -> None:
|
||||
actual = len(context.found_tests) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(f"Expected {count} test file(s), found {actual}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag removal verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I check expected fail removal for bug {bug_num:d}")
|
||||
def step_when_check_removal(context: object, bug_num: int) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
# Filter by bug tag first — matching the production path in run_quality_gate.
|
||||
test_files = find_tdd_tests(bug_num, tmp)
|
||||
context.removal_errors = check_expected_fail_removed( # type: ignore[attr-defined]
|
||||
test_files, bug_num
|
||||
)
|
||||
|
||||
|
||||
@then("there should be {count:d} removal error")
|
||||
def step_then_removal_errors_count(context: object, count: int) -> None:
|
||||
actual = len(context.removal_errors) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(
|
||||
f"Expected {count} removal error(s), got {actual}: {context.removal_errors}" # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then("there should be {count:d} removal errors")
|
||||
def step_then_removal_errors_count_plural(context: object, count: int) -> None:
|
||||
actual = len(context.removal_errors) # type: ignore[attr-defined]
|
||||
if actual != count:
|
||||
raise AssertionError(
|
||||
f"Expected {count} removal error(s), got {actual}: {context.removal_errors}" # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
|
||||
@then('the removal error should mention "{text}"')
|
||||
def step_then_removal_error_mentions(context: object, text: str) -> None:
|
||||
errors: list[str] = context.removal_errors # type: ignore[attr-defined]
|
||||
found = any(text in err for err in errors)
|
||||
if not found:
|
||||
raise AssertionError(
|
||||
f"Expected removal error mentioning '{text}', got: {errors}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full quality gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a temporary search root")
|
||||
def step_given_temp_search_root(context: object) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
_ensure_temp_dir(context)
|
||||
context.pr_diff = None # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@given('a temporary search root with file "{filepath}" containing "{content}"')
|
||||
def step_given_temp_search_root_with_file(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
context.pr_diff = None # type: ignore[attr-defined]
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given('the search root also has file "{filepath}" containing "{content}"')
|
||||
def step_given_search_root_also_has(
|
||||
context: object, filepath: str, content: str
|
||||
) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
@given("the PR diff does not remove expected fail tags")
|
||||
def step_given_pr_diff_no_expected_fail_removal(context: object) -> None:
|
||||
context.pr_diff = "" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I run the quality gate")
|
||||
def step_when_run_quality_gate(context: object) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
pr_desc: str = getattr(context, "pr_description", "")
|
||||
pr_diff: str | None = getattr(context, "pr_diff", None)
|
||||
if pr_diff is None:
|
||||
bug_refs = parse_bug_refs(pr_desc)
|
||||
pr_diff = _default_pr_diff_for_bug_refs(bug_refs, tmp)
|
||||
errors, _bug_refs = run_quality_gate(pr_desc, tmp, pr_diff=pr_diff)
|
||||
context.gate_errors = errors # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("the quality gate should pass")
|
||||
def step_then_gate_passes(context: object) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
if errors:
|
||||
raise AssertionError(f"Quality gate should pass but got errors: {errors}")
|
||||
|
||||
|
||||
@then("the quality gate should fail")
|
||||
def step_then_gate_fails(context: object) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
if not errors:
|
||||
raise AssertionError("Quality gate should fail but passed with no errors")
|
||||
|
||||
|
||||
@then('the quality gate errors should mention "{text}"')
|
||||
def step_then_gate_errors_mention(context: object, text: str) -> None:
|
||||
errors: list[str] = context.gate_errors # type: ignore[attr-defined]
|
||||
found = any(text in err for err in errors)
|
||||
if not found:
|
||||
raise AssertionError(
|
||||
f"Expected quality gate error mentioning '{text}', got: {errors}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call parse_bug_refs with a non-string argument")
|
||||
def step_when_parse_bug_refs_non_string(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
parse_bug_refs(123) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with bug number {num:d}")
|
||||
def step_when_find_tdd_tests_invalid(context: object, num: int) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(num, Path("/tmp/nonexistent"))
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with a non-Path search root")
|
||||
def step_when_find_tdd_tests_non_path(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(1, "/tmp/nonexistent") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then("a TypeError should be raised by the quality gate")
|
||||
def step_then_type_error_raised(context: object) -> None:
|
||||
exc = getattr(context, "caught_exception", None)
|
||||
if exc is None:
|
||||
raise AssertionError("Expected TypeError but no exception was raised")
|
||||
if not isinstance(exc, TypeError):
|
||||
raise AssertionError(f"Expected TypeError, got {type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
@then("a ValueError should be raised by the quality gate")
|
||||
def step_then_value_error_raised(context: object) -> None:
|
||||
exc = getattr(context, "caught_exception", None)
|
||||
if exc is None:
|
||||
raise AssertionError("Expected ValueError but no exception was raised")
|
||||
if not isinstance(exc, ValueError):
|
||||
raise AssertionError(f"Expected ValueError, got {type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Robot-format diff, unreadable files, and extra argument validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the PR diff removes expected fail for robot bug {bug_num:d}")
|
||||
def step_given_robot_diff_removes_expected_fail(context: object, bug_num: int) -> None:
|
||||
context.pr_diff = "\n".join( # type: ignore[attr-defined]
|
||||
[
|
||||
f"diff --git a/robot/bug{bug_num}.robot b/robot/bug{bug_num}.robot",
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@given("a temporary directory with an unreadable feature file for bug {bug_num:d}")
|
||||
def step_given_unreadable_feature_file(context: object, bug_num: int) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
full_path = tmp / "features" / "bug.feature"
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Write invalid UTF-8 bytes so read_text(encoding="utf-8") raises
|
||||
# UnicodeDecodeError (caught as OSError subclass). This is root-safe
|
||||
# unlike chmod(0o000) which root bypasses.
|
||||
full_path.write_bytes(
|
||||
f"@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}".encode() + b"\xff\xfe"
|
||||
)
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with a non-list argument")
|
||||
def step_when_check_expected_fail_removed_non_list(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed("not-a-list", 1) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with bug number {num:d}")
|
||||
def step_when_check_expected_fail_removed_bad_bug(context: object, num: int) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed([], num)
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool type guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call find_tdd_tests with boolean True as bug number")
|
||||
def step_when_find_tdd_tests_bool(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
find_tdd_tests(True, Path("/tmp/nonexistent")) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call check_expected_fail_removed with boolean True as bug number")
|
||||
def step_when_check_expected_fail_removed_bool(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
check_expected_fail_removed([], True) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Co-located bug false positive guard (M1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the PR diff only removes expected fail for bug {other:d} not bug {target:d}")
|
||||
def step_given_pr_diff_removes_wrong_bug(
|
||||
context: object, other: int, target: int
|
||||
) -> None:
|
||||
# Craft a diff that removes expected-fail for bug `other` but NOT for
|
||||
# bug `target`. The diff must mention bug `other`'s tag on the removed
|
||||
# line, so the gate should NOT count this as a removal for `target`.
|
||||
context.pr_diff = "\n".join( # type: ignore[attr-defined]
|
||||
[
|
||||
"diff --git a/features/bugs99.feature b/features/bugs99.feature",
|
||||
"--- a/features/bugs99.feature",
|
||||
"+++ b/features/bugs99.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{other}",
|
||||
f"+@tdd_bug @tdd_bug_{other}",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_quality_gate argument validation (L4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-string PR description")
|
||||
def step_when_run_quality_gate_non_str_desc(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate(123, Path("/tmp")) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-Path search root")
|
||||
def step_when_run_quality_gate_non_path_root(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", "/tmp") # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with an empty base_ref")
|
||||
def step_when_run_quality_gate_empty_base_ref(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", Path("/tmp"), base_ref=" ")
|
||||
except ValueError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@when("I call run_quality_gate with a non-string pr_diff")
|
||||
def step_when_run_quality_gate_non_str_diff(context: object) -> None:
|
||||
context.caught_exception = None # type: ignore[attr-defined]
|
||||
try:
|
||||
run_quality_gate("desc", Path("/tmp"), pr_diff=123) # type: ignore[arg-type]
|
||||
except TypeError as exc:
|
||||
context.caught_exception = exc # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# main() CLI entry point (M2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I call main with PR_DESCRIPTION "{description}"')
|
||||
def step_when_call_main_with_desc(context: object, description: str) -> None:
|
||||
tmp = _ensure_temp_dir(context)
|
||||
saved_desc = os.environ.get("PR_DESCRIPTION")
|
||||
saved_cwd = os.getcwd()
|
||||
try:
|
||||
os.environ["PR_DESCRIPTION"] = description
|
||||
os.chdir(tmp)
|
||||
context.main_exit_code = quality_gate_main() # type: ignore[attr-defined]
|
||||
finally:
|
||||
os.chdir(saved_cwd)
|
||||
if saved_desc is None:
|
||||
os.environ.pop("PR_DESCRIPTION", None)
|
||||
else:
|
||||
os.environ["PR_DESCRIPTION"] = saved_desc
|
||||
|
||||
|
||||
@when('I call main with PR_DESCRIPTION "{description}" in an empty search root')
|
||||
def step_when_call_main_missing_test(context: object, description: str) -> None:
|
||||
_cleanup_temp_dir(context)
|
||||
tmp = _ensure_temp_dir(context)
|
||||
saved_desc = os.environ.get("PR_DESCRIPTION")
|
||||
saved_base = os.environ.get("PR_BASE_REF")
|
||||
saved_cwd = os.getcwd()
|
||||
try:
|
||||
os.environ["PR_DESCRIPTION"] = description
|
||||
# Use a base_ref that won't exist in the temp dir (not a git repo),
|
||||
# so _collect_pr_diff will fail and produce an error about the diff.
|
||||
os.environ["PR_BASE_REF"] = "master"
|
||||
os.chdir(tmp)
|
||||
context.main_exit_code = quality_gate_main() # type: ignore[attr-defined]
|
||||
finally:
|
||||
os.chdir(saved_cwd)
|
||||
if saved_desc is None:
|
||||
os.environ.pop("PR_DESCRIPTION", None)
|
||||
else:
|
||||
os.environ["PR_DESCRIPTION"] = saved_desc
|
||||
if saved_base is None:
|
||||
os.environ.pop("PR_BASE_REF", None)
|
||||
else:
|
||||
os.environ["PR_BASE_REF"] = saved_base
|
||||
|
||||
|
||||
@then("the main exit code should be {code:d}")
|
||||
def step_then_main_exit_code(context: object, code: int) -> None:
|
||||
actual = context.main_exit_code # type: ignore[attr-defined]
|
||||
if actual != code:
|
||||
raise AssertionError(f"Expected main exit code {code}, got {actual}")
|
||||
@@ -0,0 +1,170 @@
|
||||
Feature: Structural component output validation
|
||||
|
||||
Output validation checks structural components (not exact characters).
|
||||
Required fields are validated for presence and correct type.
|
||||
Structural relationships are validated (parent-child, ordering).
|
||||
Minor formatting differences do not cause validation failures.
|
||||
Validation failures produce actionable error messages.
|
||||
|
||||
Based on Epic #8137 - structural output validation overhaul.
|
||||
|
||||
Scenario Outline: validate_plan_tree accepts valid nodes
|
||||
Given the plan tree contains {node_count:d} node(s)
|
||||
When I validate the plan tree
|
||||
Then it should be structurally valid
|
||||
And it should report no errors
|
||||
Examples:
|
||||
| node_count |
|
||||
| 1 |
|
||||
| 5 |
|
||||
| 20 |
|
||||
|
||||
Scenario: validate_plan_tree rejects nodes missing required keys
|
||||
Given the plan tree contains a node missing "decision_id"
|
||||
When I validate the plan tree
|
||||
Then it should be structurally invalid
|
||||
And it should report errors including "missing required key(s)"
|
||||
|
||||
Scenario: validate_plan_tree rejects invalid ULID in decision_id
|
||||
Given the plan tree contains a node with invalid ULID "not-a-ulid-12345"
|
||||
When I validate the plan tree
|
||||
Then it should be structurally invalid
|
||||
And it should report errors including "must be a valid ULID"
|
||||
|
||||
Scenario: validate_plan_tree accepts empty children list
|
||||
Given the plan tree contains a node with empty "children" list
|
||||
When I validate the plan tree
|
||||
Then it should be structurally valid
|
||||
|
||||
Scenario: validate_plan_tree rejects duplicate sequences for siblings
|
||||
Given two sibling nodes share the same sequence number
|
||||
When I validate the plan tree
|
||||
Then it should be structurally invalid
|
||||
And it should report errors including "duplicate sequence"
|
||||
|
||||
Scenario Outline: validate_plan_tree accepts non-negative sequences
|
||||
Given a plan tree node with sequence {seq}
|
||||
When I validate the plan tree
|
||||
Then it should be structurally valid
|
||||
Examples:
|
||||
| seq |
|
||||
| 0 |
|
||||
| 1 |
|
||||
| 100 |
|
||||
|
||||
Scenario: validate_plan_tree rejects negative sequence
|
||||
Given a plan tree node with sequence "-1"
|
||||
When I validate the plan tree
|
||||
Then it should be structurally invalid
|
||||
And error message must include "must be a non-negative integer"
|
||||
|
||||
Scenario Outline: validate_decision_dict accepts valid CLI dict for field {field}
|
||||
Given I have a decision dict with "{field}" set to {value}
|
||||
When I validate the decision dict
|
||||
Then it should be structurally valid
|
||||
Examples:
|
||||
| field | value |
|
||||
| decision_id | "01ARZ3NDEKTSV4XXFFJFRC889A"|
|
||||
| plan_id | "01ARZ3NDEKTSV4XXFFJFRC889B"|
|
||||
| type | "strategy_choice" |
|
||||
| sequence | 5 |
|
||||
| question | "Which approach?" |
|
||||
| chosen | "A" |
|
||||
| confidence | 0.75 |
|
||||
| parent | "(root)" |
|
||||
| is_correction | false |
|
||||
| superseded | false |
|
||||
|
||||
Scenario: validate_decision_dict rejects confidence outside [0,1]
|
||||
Given I have a decision dict with "confidence" set to 1.5
|
||||
When I validate the decision dict
|
||||
Then it should be structurally invalid
|
||||
And error message must include "must be in range [0.0, 1.0]"
|
||||
|
||||
Scenario: validate_decision_dict accepts None confidence
|
||||
Given I have a decision dict with "confidence" set to null
|
||||
When I validate the decision dict
|
||||
Then it should be structurally valid
|
||||
|
||||
Scenario: validate_decision_dict rejects non-string question
|
||||
Given I have a decision dict with "question" set to 42
|
||||
When I validate the decision dict
|
||||
Then it should be structurally invalid
|
||||
And error message must include "must be a non-empty string"
|
||||
|
||||
Scenario: validate_decision_dict rejects parent that is neither ULID nor "(root)"
|
||||
Given I have a decision dict with "parent" set to "random-string"
|
||||
When I validate the decision dict
|
||||
Then it should be structurally invalid
|
||||
And error message must include "must be a ULID or '(root)'"
|
||||
|
||||
Scenario: validate_decision_dict rejects is_correction that is not boolean
|
||||
Given I have a decision dict with "is_correction" set to "yes"
|
||||
When I validate the decision dict
|
||||
Then it should be structurally invalid
|
||||
And error message must include "must be a boolean"
|
||||
|
||||
Scenario Outline: validate_structured_output accepts valid envelope for field {field}
|
||||
Given I have a structured output with "{field}" set to {value} (and all other required fields present)
|
||||
When I validate the structured output
|
||||
Then it should be structurally valid
|
||||
Examples:
|
||||
| field | value |
|
||||
| command | "agents plan list" |
|
||||
| session_id| "01ARZ3NDEKTSV4XXFFJFRC889A"|
|
||||
| status | "ok" |
|
||||
| exit_code | 0 |
|
||||
|
||||
Scenario: validate_structured_output rejects invalid status value
|
||||
Given I have a structured output with "status" set to "running" (and all other fields valid)
|
||||
When I validate the structured output
|
||||
Then it should be structurally invalid
|
||||
And error must include "must be one of"
|
||||
|
||||
Scenario: validate_structured_output rejects negative exit_code
|
||||
Given I have a structured output with "exit_code" set to "-1" (and all other fields valid)
|
||||
When I validate the structured output
|
||||
Then it should be structurally invalid
|
||||
And error must include "must be >= 0"
|
||||
|
||||
Scenario: validate_structured_output accepts zero elements list
|
||||
Given I have a structured output with empty "elements" list (and all other fields valid)
|
||||
When I validate the structured output
|
||||
Then it should be structurally valid
|
||||
|
||||
Scenario: validate_structured_output rejects invalid element format
|
||||
Given I have a structured output with an element missing "kind" field
|
||||
When I validate the structured output
|
||||
Then it should be structurally invalid
|
||||
And error must include "missing 'kind' field"
|
||||
|
||||
Scenario Outline: validate_structured_component_output routes to correct validator for type {target}
|
||||
Given I target validation for "{target}" with valid data
|
||||
When I call validate_structured_component_output
|
||||
Then it should dispatch to the matching validator
|
||||
And return a valid result
|
||||
Examples:
|
||||
| target |
|
||||
| plan_tree |
|
||||
| decision |
|
||||
| structured_output|
|
||||
|
||||
Scenario: validate_structured_component_output rejects unknown target type
|
||||
Given I target validation for "unknown_type" with valid data
|
||||
When I call validate_structured_component_output
|
||||
Then it should raise a ValidationError with message containing "Unknown target_type"
|
||||
|
||||
Scenario: Structural output is robust to minor formatting differences
|
||||
Given I have a decision dict where whitespace may vary between fields
|
||||
When I validate the decision dict
|
||||
Then validation should pass (structure matched, not exact characters)
|
||||
|
||||
Scenario: Plan tree nodes preserve parent-child structural relationships
|
||||
Given a plan tree with parent and child node references connected by ULID
|
||||
When I validate the plan tree structure
|
||||
Then parent-child relationships should be validly connected
|
||||
|
||||
Scenario: Validation error messages are actionable
|
||||
Given I have invalid plan tree data
|
||||
When I validate the plan tree
|
||||
Then the first error message should identify "node[0]" and the specific field issue
|
||||
@@ -0,0 +1,33 @@
|
||||
@tdd_issue @tdd_issue_11121
|
||||
Feature: TDD Issue #11121 — cleanup_stale destroys git worktree branch on re-invoked execute
|
||||
As a developer
|
||||
I want to verify that _create_sandbox_for_plan does NOT delete the cleveragents/plan-<id>
|
||||
branch when the plan is already in execute/complete state
|
||||
So that plan apply can subsequently find and merge the correct artifacts
|
||||
|
||||
Bug #11121: _create_sandbox_for_plan() calls GitWorktreeSandbox.cleanup_stale()
|
||||
unconditionally every time agents plan execute is invoked — including when the plan
|
||||
is already in execute/complete state (execution finished, awaiting apply). This deletes
|
||||
the cleveragents/plan-<id> git branch that holds the execution output, so when
|
||||
agents plan apply runs next it finds no branch and merges zero artifacts.
|
||||
|
||||
@tdd_issue @tdd_issue_11121 @mock_only
|
||||
Scenario: cleveragents/plan-<id> branch survives a second _create_sandbox_for_plan call when plan is execute/complete
|
||||
Given a temp git repo with an execute-output branch for tdd 11121
|
||||
And a mocked plan service with the plan in execute/complete state for tdd 11121
|
||||
When I call _create_sandbox_for_plan a second time on the execute/complete plan for tdd 11121
|
||||
Then the cleveragents plan branch should still exist after the second call for tdd 11121
|
||||
|
||||
@tdd_issue @tdd_issue_11121 @mock_only
|
||||
Scenario: plan apply finds non-zero artifacts after re-invoked execute on execute/complete plan
|
||||
Given a temp git repo with an execute-output branch for tdd 11121
|
||||
And a mocked plan service with the plan in execute/complete state for tdd 11121
|
||||
When I call _create_sandbox_for_plan a second time on the execute/complete plan for tdd 11121
|
||||
Then the apply sandbox changes should find at least one artifact for tdd 11121
|
||||
|
||||
@tdd_issue @tdd_issue_11121 @mock_only
|
||||
Scenario: cleveragents/plan-<id> branch survives a second _create_sandbox_for_plan call when plan is execute/processing
|
||||
Given a temp git repo with an execute-output branch for tdd 11121
|
||||
And a mocked plan service with the plan in execute/processing state for tdd 11121
|
||||
When I call _create_sandbox_for_plan a second time on the execute/processing plan for tdd 11121
|
||||
Then the cleveragents plan branch should still exist after the second call for tdd 11121
|
||||
@@ -0,0 +1,38 @@
|
||||
@tdd_issue @tdd_issue_9131 @mock_only
|
||||
Feature: TDD Issue #9131 — invariant_enforced decisions not propagated to child plans on subplan spawn
|
||||
As a developer
|
||||
I want to verify that SubplanService.spawn() propagates invariant_enforced decisions
|
||||
from the parent plan to each child plan's decision tree
|
||||
So that child plans start Strategize with the parent's enforced invariant set
|
||||
|
||||
The spec states: "recorded as invariant_enforced decisions that propagate to child plans."
|
||||
SubplanService.spawn() currently creates child Plan objects with no invariant_enforced
|
||||
decisions, violating the spec's propagation requirement.
|
||||
|
||||
@tdd_issue @tdd_issue_9131
|
||||
Scenario: Child plan receives parent invariant_enforced decisions on spawn
|
||||
Given a parent plan with invariant_enforced decisions recorded
|
||||
And a valid spawn entry for the parent plan
|
||||
When I spawn a child plan via SubplanService
|
||||
Then the decision service should have recorded invariant_enforced decisions for the child plan
|
||||
|
||||
@tdd_issue @tdd_issue_9131
|
||||
Scenario: Child plan receives all parent invariant_enforced decisions
|
||||
Given a parent plan with 3 invariant_enforced decisions recorded
|
||||
And a valid spawn entry for the parent plan
|
||||
When I spawn a child plan via SubplanService
|
||||
Then the child plan should have 3 invariant_enforced decisions recorded
|
||||
|
||||
@tdd_issue @tdd_issue_9131
|
||||
Scenario: Non-overridable global invariants are propagated to child plans
|
||||
Given a parent plan with a non_overridable invariant_enforced decision
|
||||
And a valid spawn entry for the parent plan
|
||||
When I spawn a child plan via SubplanService
|
||||
Then the child plan should have the non_overridable invariant decision propagated
|
||||
|
||||
@tdd_issue @tdd_issue_9131
|
||||
Scenario: Child plan with no parent invariant_enforced decisions spawns cleanly
|
||||
Given a parent plan with no invariant_enforced decisions
|
||||
And a valid spawn entry for the parent plan
|
||||
When I spawn a child plan via SubplanService
|
||||
Then the spawn should succeed with no invariant decisions recorded for the child
|
||||
@@ -1,265 +0,0 @@
|
||||
Feature: TDD bug tag quality gate for bug fix PRs
|
||||
As a CI system
|
||||
I want to enforce TDD bug fix workflow rules on PRs
|
||||
So that bug fix PRs follow the required TDD workflow
|
||||
|
||||
# --- PR description parsing ---
|
||||
|
||||
Scenario: Parse single Fixes reference
|
||||
Given a PR description "Fixes #42"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
Scenario: Parse single Closes reference
|
||||
Given a PR description "Closes #100"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [100]
|
||||
|
||||
Scenario: Parse single Resolves reference
|
||||
Given a PR description "Resolves #7"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [7]
|
||||
|
||||
Scenario: Parse case-insensitive closing keywords
|
||||
Given a PR description "fixes #10 and CLOSES #20"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [10, 20]
|
||||
|
||||
Scenario: Parse ISSUES CLOSED block
|
||||
Given a PR description "ISSUES CLOSED: #5, #10"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [5, 10]
|
||||
|
||||
Scenario: Parse multiple mixed references
|
||||
Given a PR description "Fixes #42, also closes #99. ISSUES CLOSED: #7"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [7, 42, 99]
|
||||
|
||||
Scenario: No bug references returns empty list
|
||||
Given a PR description "Add new feature for users"
|
||||
When I parse the bug references
|
||||
Then the bug references should be []
|
||||
|
||||
Scenario: Deduplicate repeated bug references
|
||||
Given a PR description "Fixes #42. Also closes #42"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
Scenario: Parse past tense closing keywords
|
||||
Given a PR description "Fixed #15, Closed #20, Resolved #25"
|
||||
When I parse the bug references
|
||||
Then the bug references should be [15, 20, 25]
|
||||
|
||||
Scenario: Ignore non-closing words that end with keyword substrings
|
||||
Given a PR description "prefixes #12 and hotfixes #34"
|
||||
When I parse the bug references
|
||||
Then the bug references should be []
|
||||
|
||||
Scenario: Parse bug reference in multi-line PR description
|
||||
Given a multiline PR description
|
||||
"""
|
||||
feat(ci): implement quality gate
|
||||
|
||||
This PR adds the TDD quality gate.
|
||||
|
||||
Fixes #42
|
||||
"""
|
||||
When I parse the bug references
|
||||
Then the bug references should be [42]
|
||||
|
||||
# --- TDD test search ---
|
||||
|
||||
Scenario: Find TDD test in .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 1 test file
|
||||
|
||||
Scenario: Find TDD test in .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 1 test file
|
||||
|
||||
Scenario: Find TDD tests in both .feature and .robot files
|
||||
Given a temporary directory with a file "features/bug.feature" containing "@tdd_bug_42"
|
||||
And a temporary directory also has a file "robot/bug.robot" containing "tdd_bug_42"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 2 test files
|
||||
|
||||
Scenario: No TDD test found for bug number
|
||||
Given a temporary directory with a file "tests/other.feature" containing "@tdd_bug_99"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
Scenario: Do not match partial TDD bug tags
|
||||
Given a temporary directory with a file "tests/partial.feature" containing "@tdd_bug_420"
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
# --- Tag removal verification ---
|
||||
|
||||
Scenario: Expected fail tag still present in .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 1 removal error
|
||||
And the removal error should mention "@tdd_expected_fail"
|
||||
And the removal error should mention "@tdd_bug_42"
|
||||
|
||||
Scenario: Expected fail tag removed from .feature file
|
||||
Given a temporary directory with a file "tests/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
Scenario: Expected fail tag still present in .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_expected_fail tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 1 removal error
|
||||
|
||||
Scenario: Expected fail tag removed from .robot file
|
||||
Given a temporary directory with a file "tests/bug.robot" containing "tdd_bug tdd_bug_42"
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
# --- Full quality gate ---
|
||||
|
||||
Scenario: Quality gate passes when no bug refs in PR
|
||||
Given a temporary search root
|
||||
And a PR description "Add new feature"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate fails when no TDD test exists for referenced bug
|
||||
Given a temporary search root
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No TDD test found for bug #42"
|
||||
|
||||
Scenario: Quality gate fails when expected fail tag is still present
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "@tdd_expected_fail"
|
||||
|
||||
Scenario: Quality gate passes when expected fail tag has been removed
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate handles multiple bug references
|
||||
Given a temporary search root with file "features/bug10.feature" containing "@tdd_bug @tdd_bug_10"
|
||||
And the search root also has file "features/bug20.feature" containing "@tdd_expected_fail @tdd_bug @tdd_bug_20"
|
||||
And a PR description "Fixes #10 and fixes #20"
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "@tdd_bug_20"
|
||||
|
||||
Scenario: Quality gate passes when all bugs have clean TDD tests
|
||||
Given a temporary search root with file "features/bug10.feature" containing "@tdd_bug @tdd_bug_10"
|
||||
And the search root also has file "robot/bug20.robot" containing "tdd_bug tdd_bug_20"
|
||||
And a PR description "Fixes #10 and fixes #20"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Quality gate fails when PR diff does not remove expected fail tags
|
||||
Given a temporary search root with file "features/bug.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff does not remove expected fail tags
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No removal of @tdd_expected_fail / tdd_expected_fail detected"
|
||||
|
||||
Scenario: Quality gate passes for robot diff with expected fail removed across hunks
|
||||
Given a temporary search root with file "robot/bug.robot" containing "tdd_bug tdd_bug_42"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff removes expected fail for robot bug 42
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: Issue number zero is silently ignored
|
||||
Given a temporary search root
|
||||
And a PR description "Fixes #0"
|
||||
When I run the quality gate
|
||||
Then the quality gate should pass
|
||||
|
||||
Scenario: find_tdd_tests skips unreadable files
|
||||
Given a temporary directory with an unreadable feature file for bug 42
|
||||
When I search for TDD tests for bug 42
|
||||
Then the search should find 0 test files
|
||||
|
||||
Scenario: check_expected_fail_removed skips unreadable files
|
||||
Given a temporary directory with an unreadable feature file for bug 42
|
||||
When I check expected fail removal for bug 42
|
||||
Then there should be 0 removal errors
|
||||
|
||||
# --- Argument validation ---
|
||||
|
||||
Scenario: parse_bug_refs rejects non-string input
|
||||
When I call parse_bug_refs with a non-string argument
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: find_tdd_tests rejects invalid bug number
|
||||
When I call find_tdd_tests with bug number 0
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: find_tdd_tests rejects non-Path search root
|
||||
When I call find_tdd_tests with a non-Path search root
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects non-list input
|
||||
When I call check_expected_fail_removed with a non-list argument
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects invalid bug number
|
||||
When I call check_expected_fail_removed with bug number 0
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
# --- Bool type guard ---
|
||||
|
||||
Scenario: find_tdd_tests rejects boolean True as bug number
|
||||
When I call find_tdd_tests with boolean True as bug number
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: check_expected_fail_removed rejects boolean True as bug number
|
||||
When I call check_expected_fail_removed with boolean True as bug number
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
# --- Co-located bug false positive guard (M1) ---
|
||||
|
||||
Scenario: Diff detection does not false-positive for co-located bug tests
|
||||
Given a temporary search root with file "features/bugs.feature" containing "@tdd_bug @tdd_bug_42"
|
||||
And the search root also has file "features/bugs99.feature" containing "@tdd_bug @tdd_bug_99"
|
||||
And a PR description "Fixes #42"
|
||||
And the PR diff only removes expected fail for bug 99 not bug 42
|
||||
When I run the quality gate
|
||||
Then the quality gate should fail
|
||||
And the quality gate errors should mention "No removal of @tdd_expected_fail / tdd_expected_fail detected"
|
||||
|
||||
# --- run_quality_gate argument validation (L4) ---
|
||||
|
||||
Scenario: run_quality_gate rejects non-string PR description
|
||||
When I call run_quality_gate with a non-string PR description
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects non-Path search root
|
||||
When I call run_quality_gate with a non-Path search root
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects empty base_ref
|
||||
When I call run_quality_gate with an empty base_ref
|
||||
Then a ValueError should be raised by the quality gate
|
||||
|
||||
Scenario: run_quality_gate rejects non-string pr_diff
|
||||
When I call run_quality_gate with a non-string pr_diff
|
||||
Then a TypeError should be raised by the quality gate
|
||||
|
||||
# --- main() CLI entry point (M2) ---
|
||||
|
||||
Scenario: main returns 0 when no bug refs in PR description
|
||||
When I call main with PR_DESCRIPTION "Add new feature"
|
||||
Then the main exit code should be 0
|
||||
|
||||
Scenario: main returns 1 when TDD test is missing
|
||||
When I call main with PR_DESCRIPTION "Fixes #99999" in an empty search root
|
||||
Then the main exit code should be 1
|
||||
-23
@@ -882,29 +882,6 @@ def benchmark_regression(session: nox.Session):
|
||||
session.run("asv", "publish", f"--config={config_path}")
|
||||
|
||||
|
||||
@nox.session(python=DEFAULT_PYTHON, reuse_venv=True, venv_backend="uv")
|
||||
def tdd_quality_gate(session: nox.Session):
|
||||
"""Enforce TDD bug fix workflow rules on PRs.
|
||||
|
||||
Reads the PR description from the ``PR_DESCRIPTION`` environment
|
||||
variable and verifies that:
|
||||
|
||||
1. Every bug referenced via closing keywords (``Fixes #N``,
|
||||
``Closes #N``, ``Resolves #N``, ``ISSUES CLOSED: #N``) has
|
||||
a corresponding TDD test tagged ``@tdd_bug_N``.
|
||||
2. The ``@tdd_expected_fail`` / ``tdd_expected_fail`` tag has been
|
||||
removed from each of those tests in the PR diff.
|
||||
|
||||
If no bug references are found the gate passes trivially.
|
||||
"""
|
||||
# The quality gate script uses only standard library; no install needed.
|
||||
pr_description = os.environ.get("PR_DESCRIPTION", "")
|
||||
pr_base_ref = os.environ.get("PR_BASE_REF", "master")
|
||||
session.env["PR_DESCRIPTION"] = pr_description
|
||||
session.env["PR_BASE_REF"] = pr_base_ref
|
||||
session.run("python", "scripts/tdd_quality_gate.py")
|
||||
|
||||
|
||||
# Sessions to run by default when running `nox` without arguments
|
||||
nox.options.sessions = [
|
||||
"lint", # ~5-10 seconds
|
||||
|
||||
@@ -1,412 +0,0 @@
|
||||
"""Helper for ``tdd_quality_gate.robot`` — exercises TDD quality gate logic.
|
||||
|
||||
Each sub-command exercises a specific aspect of the quality gate and
|
||||
prints a sentinel string on success. Exit 0 = check passed,
|
||||
1 = unexpected outcome.
|
||||
|
||||
See CONTRIBUTING.md > Bug Fix Workflow for the full specification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure the project root is importable.
|
||||
_ROOT = str(Path(__file__).resolve().parents[1])
|
||||
if _ROOT not in sys.path:
|
||||
sys.path.insert(0, _ROOT)
|
||||
|
||||
from scripts.tdd_quality_gate import ( # noqa: E402
|
||||
find_tdd_tests,
|
||||
parse_bug_refs,
|
||||
run_quality_gate,
|
||||
)
|
||||
|
||||
|
||||
def _make_temp_tree(files: dict[str, str]) -> Path:
|
||||
"""Create a temporary directory with the given file tree."""
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
for filepath, content in files.items():
|
||||
full_path = tmp / filepath
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
full_path.write_text(content, encoding="utf-8")
|
||||
return tmp
|
||||
|
||||
|
||||
def _default_pr_diff_for_bug_refs(
|
||||
bug_refs: list[int], search_root: Path | None = None
|
||||
) -> str:
|
||||
"""Return a synthetic PR diff that removes expected-fail tags.
|
||||
|
||||
When *search_root* is provided the helper inspects the temp tree to
|
||||
decide whether each bug's TDD test lives in a ``.feature`` or
|
||||
``.robot`` file and emits the diff in the matching format.
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
for bug_num in bug_refs:
|
||||
use_robot = False
|
||||
if search_root is not None:
|
||||
robot_hits = list(search_root.rglob("*.robot"))
|
||||
for rp in robot_hits:
|
||||
try:
|
||||
if f"tdd_bug_{bug_num}" in rp.read_text(encoding="utf-8"):
|
||||
use_robot = True
|
||||
break
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
if use_robot:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
(
|
||||
f"diff --git a/robot/bug{bug_num}.robot "
|
||||
f"b/robot/bug{bug_num}.robot"
|
||||
),
|
||||
f"--- a/robot/bug{bug_num}.robot",
|
||||
f"+++ b/robot/bug{bug_num}.robot",
|
||||
"@@ -1 +1 @@",
|
||||
f"-tdd_expected_fail tdd_bug tdd_bug_{bug_num}",
|
||||
f"+tdd_bug tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
else:
|
||||
chunks.append(
|
||||
"\n".join(
|
||||
[
|
||||
(
|
||||
f"diff --git a/features/bug{bug_num}.feature "
|
||||
f"b/features/bug{bug_num}.feature"
|
||||
),
|
||||
f"--- a/features/bug{bug_num}.feature",
|
||||
f"+++ b/features/bug{bug_num}.feature",
|
||||
"@@ -1 +1 @@",
|
||||
f"-@tdd_expected_fail @tdd_bug @tdd_bug_{bug_num}",
|
||||
f"+@tdd_bug @tdd_bug_{bug_num}",
|
||||
]
|
||||
)
|
||||
)
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_fixes_single() -> int:
|
||||
"""Verify parsing a single Fixes #N reference."""
|
||||
refs = parse_bug_refs("Fixes #42")
|
||||
if refs != [42]:
|
||||
print(f"FAIL: expected [42], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-fixes-single-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_mixed_refs() -> int:
|
||||
"""Verify parsing multiple mixed closing keywords."""
|
||||
refs = parse_bug_refs("Fixes #42, also closes #99. Resolves #7")
|
||||
if refs != [7, 42, 99]:
|
||||
print(f"FAIL: expected [7, 42, 99], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-mixed-refs-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_issues_closed() -> int:
|
||||
"""Verify parsing ISSUES CLOSED: block."""
|
||||
refs = parse_bug_refs("ISSUES CLOSED: #5, #10")
|
||||
if refs != [5, 10]:
|
||||
print(f"FAIL: expected [5, 10], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-issues-closed-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_non_closing_words() -> int:
|
||||
"""Verify parser ignores embedded keyword substrings."""
|
||||
refs = parse_bug_refs("prefixes #12 and hotfixes #34")
|
||||
if refs != []:
|
||||
print(f"FAIL: expected [], got {refs}", file=sys.stderr)
|
||||
return 1
|
||||
print("parse-non-closing-words-ok")
|
||||
return 0
|
||||
|
||||
|
||||
def no_bug_refs_pass() -> int:
|
||||
"""Verify the quality gate passes when no bug refs in PR."""
|
||||
tmp = _make_temp_tree({})
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Add new feature", tmp)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("no-bug-refs-pass-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_feature_test() -> int:
|
||||
"""Verify finding @tdd_bug_N in .feature files."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 1:
|
||||
print(f"FAIL: expected 1 test, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-feature-test-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def find_robot_test() -> int:
|
||||
"""Verify finding tdd_bug_N in .robot files."""
|
||||
tmp = _make_temp_tree({"robot/bug.robot": "[Tags] tdd_bug tdd_bug_42\n"})
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 1:
|
||||
print(f"FAIL: expected 1 test, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-robot-test-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def find_exact_tag_match() -> int:
|
||||
"""Verify partial tags are not treated as exact bug tag matches."""
|
||||
tmp = _make_temp_tree({"features/partial.feature": "@tdd_bug_420\nFeature: Test\n"})
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if tests:
|
||||
print(f"FAIL: expected 0 tests, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
print("find-exact-tag-match-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality gate integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def no_tdd_test_fails() -> int:
|
||||
"""Verify the gate fails when no TDD test exists for a referenced bug."""
|
||||
tmp = _make_temp_tree({})
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff="")
|
||||
if not errors:
|
||||
print("FAIL: expected errors for missing TDD test", file=sys.stderr)
|
||||
return 1
|
||||
if not any("No TDD test found for bug #42" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected 'No TDD test found' error, got: {errors}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("no-tdd-test-fails-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def expected_fail_present() -> int:
|
||||
"""Verify the gate fails when @tdd_expected_fail is still present."""
|
||||
ef_content = "@tdd_expected_fail @tdd_bug @tdd_bug_42\nFeature: Test\n"
|
||||
tmp = _make_temp_tree({"features/bug.feature": ef_content})
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if not errors:
|
||||
print(
|
||||
"FAIL: expected errors for @tdd_expected_fail present", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
if not any("@tdd_expected_fail" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected '@tdd_expected_fail' error, got: {errors}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("expected-fail-present-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def expected_fail_removed() -> int:
|
||||
"""Verify the gate passes when @tdd_expected_fail has been removed."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("expected-fail-removed-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def multi_bug_mixed() -> int:
|
||||
"""Verify the gate handles multiple bugs with mixed outcomes."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug10.feature": "@tdd_bug @tdd_bug_10\nFeature: Bug 10\n",
|
||||
"features/bug20.feature": (
|
||||
"@tdd_expected_fail @tdd_bug @tdd_bug_20\nFeature: Bug 20\n"
|
||||
),
|
||||
}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([10, 20], tmp)
|
||||
errors, _refs = run_quality_gate(
|
||||
"Fixes #10 and fixes #20", tmp, pr_diff=pr_diff
|
||||
)
|
||||
if not errors:
|
||||
print("FAIL: expected errors for bug #20", file=sys.stderr)
|
||||
return 1
|
||||
if not any("@tdd_bug_20" in e for e in errors):
|
||||
print(f"FAIL: expected error about bug #20, got: {errors}", file=sys.stderr)
|
||||
return 1
|
||||
# Bug #10 should not have errors
|
||||
if any("@tdd_bug_10" in e for e in errors):
|
||||
print(f"FAIL: unexpected error about bug #10: {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("multi-bug-mixed-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def all_clean_passes() -> int:
|
||||
"""Verify the gate passes when all bugs have clean TDD tests."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug10.feature": "@tdd_bug @tdd_bug_10\nFeature: Bug 10\n",
|
||||
"robot/bug20.robot": "[Tags] tdd_bug tdd_bug_20\n",
|
||||
}
|
||||
)
|
||||
try:
|
||||
pr_diff = _default_pr_diff_for_bug_refs([10, 20], tmp)
|
||||
errors, _refs = run_quality_gate(
|
||||
"Fixes #10 and fixes #20", tmp, pr_diff=pr_diff
|
||||
)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("all-clean-passes-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def both_behave_and_robot() -> int:
|
||||
"""Verify the gate checks tests in both .feature and .robot files."""
|
||||
tmp = _make_temp_tree(
|
||||
{
|
||||
"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Bug\n",
|
||||
"robot/bug.robot": "[Tags] tdd_bug tdd_bug_42\n",
|
||||
}
|
||||
)
|
||||
try:
|
||||
tests = find_tdd_tests(42, tmp)
|
||||
if len(tests) != 2:
|
||||
print(f"FAIL: expected 2 tests, found {len(tests)}", file=sys.stderr)
|
||||
return 1
|
||||
pr_diff = _default_pr_diff_for_bug_refs([42], tmp)
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff=pr_diff)
|
||||
if errors:
|
||||
print(f"FAIL: expected no errors, got {errors}", file=sys.stderr)
|
||||
return 1
|
||||
print("both-behave-and-robot-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def diff_removal_required() -> int:
|
||||
"""Verify the gate fails when PR diff has no expected-fail removal."""
|
||||
tmp = _make_temp_tree(
|
||||
{"features/bug.feature": "@tdd_bug @tdd_bug_42\nFeature: Test\n"}
|
||||
)
|
||||
try:
|
||||
errors, _refs = run_quality_gate("Fixes #42", tmp, pr_diff="")
|
||||
if not errors:
|
||||
print("FAIL: expected diff-removal error", file=sys.stderr)
|
||||
return 1
|
||||
if not any("No removal of @tdd_expected_fail" in e for e in errors):
|
||||
print(
|
||||
f"FAIL: expected diff-removal message, got: {errors}", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
print("diff-removal-required-ok")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, Callable[[], int]] = {
|
||||
"parse_fixes_single": parse_fixes_single,
|
||||
"parse_mixed_refs": parse_mixed_refs,
|
||||
"parse_issues_closed": parse_issues_closed,
|
||||
"parse_non_closing_words": parse_non_closing_words,
|
||||
"no_bug_refs_pass": no_bug_refs_pass,
|
||||
"find_feature_test": find_feature_test,
|
||||
"find_robot_test": find_robot_test,
|
||||
"find_exact_tag_match": find_exact_tag_match,
|
||||
"no_tdd_test_fails": no_tdd_test_fails,
|
||||
"expected_fail_present": expected_fail_present,
|
||||
"expected_fail_removed": expected_fail_removed,
|
||||
"multi_bug_mixed": multi_bug_mixed,
|
||||
"all_clean_passes": all_clean_passes,
|
||||
"both_behave_and_robot": both_behave_and_robot,
|
||||
"diff_removal_required": diff_removal_required,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Dispatch to the sub-command named in sys.argv[1]."""
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <command>", file=sys.stderr)
|
||||
print(f"Commands: {sorted(_COMMANDS)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
cmd = sys.argv[1]
|
||||
handler = _COMMANDS.get(cmd)
|
||||
if handler is None:
|
||||
print(f"Unknown command: {cmd}", file=sys.stderr)
|
||||
print(f"Available: {sorted(_COMMANDS)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
return handler()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -83,9 +83,10 @@ Plan Generation Graph Builds Workflow With Correct Nodes
|
||||
... assert 'analyze_requirements' in nodes
|
||||
... assert 'generate_plan' in nodes
|
||||
... assert 'validate' in nodes
|
||||
... assert 'handle_retry' in nodes
|
||||
... print(f'Graph has {len(nodes)} nodes')
|
||||
${result}= Run Process ${PYTHON} -c ${script} shell=True
|
||||
Should Contain ${result.stdout} Graph has 4 nodes
|
||||
Should Contain ${result.stdout} Graph has 5 nodes
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
|
||||
LangGraph Graphs Package Exports Workflow Classes
|
||||
@@ -192,7 +193,9 @@ Load Context Node Generates Summary With Sample Contexts
|
||||
|
||||
|
||||
Should Retry Returns Retry When Validation Fails And Retries Available
|
||||
[Documentation] Test _should_retry returns "retry" appropriately
|
||||
[Documentation] Test _should_retry returns "retry" appropriately.
|
||||
... Note: _should_retry is a read-only conditional-edge function in LangGraph.
|
||||
... It does NOT mutate state; retry_count is incremented inside _validate.
|
||||
${script}= Catenate SEPARATOR=\n
|
||||
... import sys
|
||||
... sys.path.insert(0, '${SRC_DIR}')
|
||||
@@ -205,7 +208,7 @@ Should Retry Returns Retry When Validation Fails And Retries Available
|
||||
... }
|
||||
... decision = graph._should_retry(state)
|
||||
... assert decision == 'retry'
|
||||
... assert state['retry_count'] == 1
|
||||
... assert state['retry_count'] == 0, f'_should_retry must not mutate state, got {state["retry_count"]}'
|
||||
... print('Should retry: retry decision correct')
|
||||
${result}= Run Process ${PYTHON} -c ${script} shell=True
|
||||
Should Contain ${result.stdout} Should retry: retry decision correct
|
||||
@@ -231,7 +234,9 @@ Should Retry Returns End When Validation Passes
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
|
||||
Should Retry Returns End When Max Retries Reached
|
||||
[Documentation] Test _should_retry returns "end" when max retries reached
|
||||
[Documentation] Test _should_retry returns "end" when max retries exceeded.
|
||||
... Because _validate increments retry_count before _should_retry is called,
|
||||
... the boundary is retry_count > max_retries (i.e. retry_count=4 for max_retries=3).
|
||||
${script}= Catenate SEPARATOR=\n
|
||||
... import sys
|
||||
... sys.path.insert(0, '${SRC_DIR}')
|
||||
@@ -240,7 +245,7 @@ Should Retry Returns End When Max Retries Reached
|
||||
... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3), max_retries=3)
|
||||
... state = {
|
||||
... 'validation_result': {'status': 'FAIL'},
|
||||
... 'retry_count': 3
|
||||
... 'retry_count': 4
|
||||
... }
|
||||
... decision = graph._should_retry(state)
|
||||
... assert decision == 'end'
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
*** Settings ***
|
||||
Documentation Integration tests for the TDD quality gate script.
|
||||
...
|
||||
... Exercises ``scripts/tdd_quality_gate.py`` end-to-end by
|
||||
... invoking it as a subprocess with various PR_DESCRIPTION
|
||||
... values and temporary file trees, then verifying exit codes
|
||||
... and output messages.
|
||||
...
|
||||
... See CONTRIBUTING.md > Bug Fix Workflow for the full
|
||||
... specification.
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_tdd_quality_gate.py
|
||||
|
||||
*** Test Cases ***
|
||||
# ===========================================================================
|
||||
# PR description parsing
|
||||
# ===========================================================================
|
||||
|
||||
Parse Single Fixes Reference
|
||||
[Documentation] Verify the script extracts a single Fixes #N reference
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_fixes_single
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-fixes-single-ok
|
||||
|
||||
Parse Multiple Mixed References
|
||||
[Documentation] Verify the script extracts multiple closing keywords
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_mixed_refs
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-mixed-refs-ok
|
||||
|
||||
Parse Issues Closed Block
|
||||
[Documentation] Verify the script extracts ISSUES CLOSED: #N references
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_issues_closed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-issues-closed-ok
|
||||
|
||||
Parse Non Closing Words
|
||||
[Documentation] Verify parser ignores embedded keyword substrings
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} parse_non_closing_words
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} parse-non-closing-words-ok
|
||||
|
||||
No Bug References Passes Trivially
|
||||
[Documentation] Verify the quality gate passes when no bug refs in PR
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} no_bug_refs_pass
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} no-bug-refs-pass-ok
|
||||
|
||||
# ===========================================================================
|
||||
# TDD test search and tag verification
|
||||
# ===========================================================================
|
||||
|
||||
Find TDD Test In Feature File
|
||||
[Documentation] Verify the script finds @tdd_bug_N in .feature files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_feature_test
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-feature-test-ok
|
||||
|
||||
Find TDD Test In Robot File
|
||||
[Documentation] Verify the script finds tdd_bug_N in .robot files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_robot_test
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-robot-test-ok
|
||||
|
||||
Find TDD Test Uses Exact Tag Match
|
||||
[Documentation] Verify partial tags are not matched as exact bug tags
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} find_exact_tag_match
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} find-exact-tag-match-ok
|
||||
|
||||
No TDD Test Found Fails Gate
|
||||
[Documentation] Verify the gate fails when no TDD test exists
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} no_tdd_test_fails
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} no-tdd-test-fails-ok
|
||||
|
||||
Expected Fail Tag Still Present Fails Gate
|
||||
[Documentation] Verify the gate fails when @tdd_expected_fail is still present
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} expected_fail_present
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} expected-fail-present-ok
|
||||
|
||||
Expected Fail Tag Removed Passes Gate
|
||||
[Documentation] Verify the gate passes when @tdd_expected_fail is removed
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} expected_fail_removed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} expected-fail-removed-ok
|
||||
|
||||
# ===========================================================================
|
||||
# Full quality gate integration
|
||||
# ===========================================================================
|
||||
|
||||
Full Gate Multiple Bugs Mixed Results
|
||||
[Documentation] Verify the gate handles multiple bugs with mixed outcomes
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} multi_bug_mixed
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} multi-bug-mixed-ok
|
||||
|
||||
Full Gate All Clean Passes
|
||||
[Documentation] Verify the gate passes when all referenced bugs have clean tests
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} all_clean_passes
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} all-clean-passes-ok
|
||||
|
||||
TDD Tests In Both Behave And Robot
|
||||
[Documentation] Verify the gate checks tests in both .feature and .robot files
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} both_behave_and_robot
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} both-behave-and-robot-ok
|
||||
|
||||
Diff Removal Is Required
|
||||
[Documentation] Verify the gate fails when PR diff has no expected-fail removal
|
||||
[Tags] ci quality tdd
|
||||
${result}= Run Process ${PYTHON} ${HELPER} diff_removal_required
|
||||
... cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0 msg=${result.stderr}
|
||||
Should Contain ${result.stdout} diff-removal-required-ok
|
||||
@@ -1,402 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""TDD bug tag quality gate for bug fix PRs.
|
||||
|
||||
Enforces the TDD bug fix workflow rules described in CONTRIBUTING.md:
|
||||
|
||||
1. Parses the PR description for closing keywords that reference bug issues
|
||||
(``Closes #N``, ``Fixes #N``, ``Resolves #N``, ``ISSUES CLOSED: #N``).
|
||||
2. Searches the codebase for tests tagged ``@tdd_bug_N`` (Behave ``.feature``
|
||||
files) or ``tdd_bug_N`` (Robot ``.robot`` files).
|
||||
3. Verifies that every such test has had its ``@tdd_expected_fail`` /
|
||||
``tdd_expected_fail`` tag removed — the fix PR must remove the
|
||||
expected-fail marker as proof the bug is now fixed.
|
||||
|
||||
Exit codes:
|
||||
0 — All checks passed (or PR references no bugs).
|
||||
1 — One or more violations detected.
|
||||
|
||||
Usage:
|
||||
PR_DESCRIPTION="Fixes #42" python scripts/tdd_quality_gate.py
|
||||
|
||||
Or via nox::
|
||||
|
||||
PR_DESCRIPTION="Fixes #42" nox -s tdd_quality_gate
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PR description parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Matches: Closes #N, Fixes #N, Resolves #N (case-insensitive)
|
||||
_CLOSING_KEYWORD_RE = re.compile(
|
||||
r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\b\s+#(\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches: ISSUES CLOSED: #N, #M, ...
|
||||
_ISSUES_CLOSED_RE = re.compile(
|
||||
r"ISSUES\s+CLOSED\s*:\s*((?:#\d+[\s,]*)+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Extracts individual issue numbers from the ISSUES CLOSED value
|
||||
_ISSUE_NUMBER_RE = re.compile(r"#(\d+)")
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=64)
|
||||
def _tag_token_pattern(tag: str) -> re.Pattern[str]:
|
||||
"""Return a compiled regex that matches ``tag`` as a full token."""
|
||||
escaped = re.escape(tag)
|
||||
return re.compile(rf"(?<![A-Za-z0-9_]){escaped}(?![A-Za-z0-9_])")
|
||||
|
||||
|
||||
def _contains_tag_token(content: str, tag: str) -> bool:
|
||||
"""Return True when ``tag`` appears as a full token in ``content``."""
|
||||
return _tag_token_pattern(tag).search(content) is not None
|
||||
|
||||
|
||||
def _collect_pr_diff(search_root: Path, base_ref: str) -> str:
|
||||
"""Collect unified diff between the PR branch and the base branch."""
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
if not isinstance(base_ref, str):
|
||||
raise TypeError(f"base_ref must be a str, got {type(base_ref).__name__}")
|
||||
if base_ref.strip() == "":
|
||||
raise ValueError("base_ref must not be empty")
|
||||
|
||||
ranges = (f"origin/{base_ref}...HEAD", f"{base_ref}...HEAD")
|
||||
for ref_range in ranges:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-color",
|
||||
ref_range,
|
||||
"--",
|
||||
"*.feature",
|
||||
"*.robot",
|
||||
],
|
||||
cwd=search_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return proc.stdout
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
continue
|
||||
|
||||
raise RuntimeError(
|
||||
"Unable to compute PR diff against base branch. "
|
||||
"Ensure git history for the base branch is available and retry."
|
||||
)
|
||||
|
||||
|
||||
def _diff_has_expected_fail_removal_for_bug(pr_diff: str, bug_number: int) -> bool:
|
||||
"""Return True when PR diff removes expected-fail for ``bug_number``."""
|
||||
if not isinstance(pr_diff, str):
|
||||
raise TypeError(f"pr_diff must be a str, got {type(pr_diff).__name__}")
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
|
||||
current_suffix = ""
|
||||
in_hunk = False
|
||||
file_has_bug_tag = False
|
||||
file_removed_expected_fail = False
|
||||
|
||||
for line in pr_diff.splitlines():
|
||||
if line.startswith("+++ "):
|
||||
if file_has_bug_tag and file_removed_expected_fail:
|
||||
return True
|
||||
raw_path = line[4:]
|
||||
if raw_path.startswith("b/"):
|
||||
raw_path = raw_path[2:]
|
||||
current_suffix = Path(raw_path).suffix.lower()
|
||||
in_hunk = False
|
||||
file_has_bug_tag = False
|
||||
file_removed_expected_fail = False
|
||||
continue
|
||||
|
||||
if current_suffix not in {".feature", ".robot"}:
|
||||
continue
|
||||
|
||||
if line.startswith("@@"):
|
||||
in_hunk = True
|
||||
continue
|
||||
|
||||
if not in_hunk:
|
||||
continue
|
||||
|
||||
if not line or line[0] not in {" ", "+", "-"}:
|
||||
continue
|
||||
|
||||
content = line[1:]
|
||||
if current_suffix == ".feature":
|
||||
bug_tag = f"@tdd_bug_{bug_number}"
|
||||
expected_fail_tag = "@tdd_expected_fail"
|
||||
else:
|
||||
bug_tag = f"tdd_bug_{bug_number}"
|
||||
expected_fail_tag = "tdd_expected_fail"
|
||||
|
||||
if _contains_tag_token(content, bug_tag):
|
||||
file_has_bug_tag = True
|
||||
if (
|
||||
line[0] == "-"
|
||||
and _contains_tag_token(content, expected_fail_tag)
|
||||
and _contains_tag_token(content, bug_tag)
|
||||
):
|
||||
file_removed_expected_fail = True
|
||||
|
||||
return file_has_bug_tag and file_removed_expected_fail
|
||||
|
||||
|
||||
def parse_bug_refs(pr_description: str) -> list[int]:
|
||||
"""Extract bug issue numbers from PR closing keywords.
|
||||
|
||||
Recognises ``Closes #N``, ``Fixes #N``, ``Resolves #N``
|
||||
(case-insensitive) and ``ISSUES CLOSED: #N, #M``.
|
||||
|
||||
Returns a deduplicated, sorted list of issue numbers.
|
||||
"""
|
||||
if not isinstance(pr_description, str):
|
||||
raise TypeError(
|
||||
f"pr_description must be a str, got {type(pr_description).__name__}"
|
||||
)
|
||||
|
||||
refs: set[int] = set()
|
||||
|
||||
# Standard closing keywords
|
||||
for match in _CLOSING_KEYWORD_RE.finditer(pr_description):
|
||||
num = int(match.group(1))
|
||||
if num > 0:
|
||||
refs.add(num)
|
||||
|
||||
# ISSUES CLOSED: #N, #M block
|
||||
for block_match in _ISSUES_CLOSED_RE.finditer(pr_description):
|
||||
block = block_match.group(1)
|
||||
for num_match in _ISSUE_NUMBER_RE.finditer(block):
|
||||
num = int(num_match.group(1))
|
||||
if num > 0:
|
||||
refs.add(num)
|
||||
|
||||
return sorted(refs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD test search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_tdd_tests(
|
||||
bug_number: int,
|
||||
search_root: Path,
|
||||
) -> list[Path]:
|
||||
"""Find test files tagged with ``@tdd_bug_<bug_number>``.
|
||||
|
||||
Searches ``.feature`` files for ``@tdd_bug_<N>`` and ``.robot``
|
||||
files for ``tdd_bug_<N>``.
|
||||
|
||||
Returns a list of paths that contain the tag.
|
||||
"""
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
|
||||
tag_behave = f"@tdd_bug_{bug_number}"
|
||||
tag_robot = f"tdd_bug_{bug_number}"
|
||||
matches: list[Path] = []
|
||||
|
||||
# Search .feature files
|
||||
for feature_file in sorted(search_root.rglob("*.feature")):
|
||||
try:
|
||||
content = feature_file.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if _contains_tag_token(content, tag_behave):
|
||||
matches.append(feature_file)
|
||||
|
||||
# Search .robot files
|
||||
for robot_file in sorted(search_root.rglob("*.robot")):
|
||||
try:
|
||||
content = robot_file.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
if _contains_tag_token(content, tag_robot):
|
||||
matches.append(robot_file)
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag removal verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def check_expected_fail_removed(
|
||||
test_files: list[Path],
|
||||
bug_number: int,
|
||||
) -> list[str]:
|
||||
"""Verify ``@tdd_expected_fail`` has been removed from test files.
|
||||
|
||||
Returns a list of error messages for files that still contain the
|
||||
expected-fail tag.
|
||||
"""
|
||||
if not isinstance(test_files, list):
|
||||
raise TypeError(f"test_files must be a list, got {type(test_files).__name__}")
|
||||
if (
|
||||
isinstance(bug_number, bool)
|
||||
or not isinstance(bug_number, int)
|
||||
or bug_number < 1
|
||||
):
|
||||
raise ValueError(f"bug_number must be a positive integer, got {bug_number!r}")
|
||||
|
||||
errors: list[str] = []
|
||||
|
||||
for path in test_files:
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".feature":
|
||||
tag = "@tdd_expected_fail"
|
||||
elif suffix == ".robot":
|
||||
tag = "tdd_expected_fail"
|
||||
else:
|
||||
continue
|
||||
|
||||
if _contains_tag_token(content, tag):
|
||||
bug_tag_display = (
|
||||
f"@tdd_bug_{bug_number}"
|
||||
if suffix == ".feature"
|
||||
else f"tdd_bug_{bug_number}"
|
||||
)
|
||||
errors.append(
|
||||
f"Bug fix PR must remove the {tag} tag from tests tagged "
|
||||
f"{bug_tag_display}. "
|
||||
f"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main gate logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_quality_gate(
|
||||
pr_description: str,
|
||||
search_root: Path,
|
||||
*,
|
||||
pr_diff: str | None = None,
|
||||
base_ref: str = "master",
|
||||
) -> tuple[list[str], list[int]]:
|
||||
"""Run the full TDD quality gate.
|
||||
|
||||
Returns a ``(errors, bug_refs)`` tuple. An empty error list means all
|
||||
checks passed. ``bug_refs`` contains the parsed bug issue numbers so
|
||||
callers can avoid re-parsing the PR description.
|
||||
"""
|
||||
if not isinstance(pr_description, str):
|
||||
raise TypeError(
|
||||
f"pr_description must be a str, got {type(pr_description).__name__}"
|
||||
)
|
||||
if not isinstance(search_root, Path):
|
||||
raise TypeError(f"search_root must be a Path, got {type(search_root).__name__}")
|
||||
if pr_diff is not None and not isinstance(pr_diff, str):
|
||||
raise TypeError(f"pr_diff must be a str or None, got {type(pr_diff).__name__}")
|
||||
if not isinstance(base_ref, str):
|
||||
raise TypeError(f"base_ref must be a str, got {type(base_ref).__name__}")
|
||||
if base_ref.strip() == "":
|
||||
raise ValueError("base_ref must not be empty")
|
||||
|
||||
bug_refs = parse_bug_refs(pr_description)
|
||||
if not bug_refs:
|
||||
return [], bug_refs
|
||||
|
||||
if pr_diff is None:
|
||||
try:
|
||||
pr_diff = _collect_pr_diff(search_root, base_ref)
|
||||
except RuntimeError as exc:
|
||||
return [str(exc)], bug_refs
|
||||
|
||||
all_errors: list[str] = []
|
||||
|
||||
for bug_num in bug_refs:
|
||||
test_files = find_tdd_tests(bug_num, search_root)
|
||||
|
||||
if not test_files:
|
||||
all_errors.append(
|
||||
f"No TDD test found for bug #{bug_num}. "
|
||||
f"The TDD workflow requires a test tagged @tdd_bug_{bug_num} "
|
||||
f"to exist before the bug can be fixed. "
|
||||
f"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
continue
|
||||
|
||||
removal_errors = check_expected_fail_removed(test_files, bug_num)
|
||||
all_errors.extend(removal_errors)
|
||||
|
||||
# Only check the diff when the file-level check found no tag issues;
|
||||
# otherwise the diff error would be redundant.
|
||||
if not removal_errors and not _diff_has_expected_fail_removal_for_bug(
|
||||
pr_diff, bug_num
|
||||
):
|
||||
all_errors.append(
|
||||
"No removal of @tdd_expected_fail / tdd_expected_fail detected "
|
||||
f"in PR diff for bug #{bug_num}. "
|
||||
"The bug fix PR must remove the expected-fail tag in this branch. "
|
||||
"See CONTRIBUTING.md > Bug Fix Workflow."
|
||||
)
|
||||
|
||||
return all_errors, bug_refs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point. Reads PR_DESCRIPTION from the environment."""
|
||||
pr_description = os.environ.get("PR_DESCRIPTION", "")
|
||||
base_ref = os.environ.get("PR_BASE_REF", "master")
|
||||
search_root = Path.cwd()
|
||||
|
||||
errors, bug_refs = run_quality_gate(pr_description, search_root, base_ref=base_ref)
|
||||
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if bug_refs:
|
||||
print(f"TDD quality gate passed for bug(s): {bug_refs}")
|
||||
else:
|
||||
print("TDD quality gate: no bug references found in PR description (pass).")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -154,6 +154,8 @@ class PlanGenerationGraph:
|
||||
2. analyze_requirements: Analyzes user prompt for requirements
|
||||
3. generate_plan: Generates code changes based on requirements
|
||||
4. validate: Validates generated changes
|
||||
5. handle_retry: Increments the retry counter (bridges conditional edge to
|
||||
state update, since LangGraph conditional edges cannot persist mutations)
|
||||
|
||||
The workflow includes conditional edges for retry logic and checkpointing
|
||||
for resumable execution.
|
||||
@@ -273,6 +275,7 @@ class PlanGenerationGraph:
|
||||
workflow.add_node("analyze_requirements", self._analyze_requirements)
|
||||
workflow.add_node("generate_plan", self._generate_plan)
|
||||
workflow.add_node("validate", self._validate)
|
||||
workflow.add_node("handle_retry", self._handle_retry)
|
||||
|
||||
# Set entry point
|
||||
workflow.set_entry_point("load_context")
|
||||
@@ -282,12 +285,16 @@ class PlanGenerationGraph:
|
||||
workflow.add_edge("analyze_requirements", "generate_plan")
|
||||
workflow.add_edge("generate_plan", "validate")
|
||||
|
||||
# Route retries through handle_retry node to persist the retry_count
|
||||
# increment (conditional edge functions cannot mutate state).
|
||||
workflow.add_edge("handle_retry", "analyze_requirements")
|
||||
|
||||
# Add conditional edge for retry logic
|
||||
workflow.add_conditional_edges(
|
||||
"validate",
|
||||
self._should_retry,
|
||||
{
|
||||
"retry": "analyze_requirements",
|
||||
"retry": "handle_retry",
|
||||
"end": END,
|
||||
},
|
||||
)
|
||||
@@ -497,9 +504,15 @@ class PlanGenerationGraph:
|
||||
state: Current workflow state
|
||||
|
||||
Returns:
|
||||
Updated state with validation results
|
||||
Updated state with validation results and incremented retry_count.
|
||||
The retry_count is incremented here (inside a node) so that
|
||||
LangGraph persists the new value into the graph state. It must
|
||||
NOT be mutated inside ``_should_retry`` because conditional-edge
|
||||
functions are read-only from LangGraph's perspective — any
|
||||
mutations they make to the state dict are silently discarded.
|
||||
"""
|
||||
changes = state.get("generated_changes", [])
|
||||
retry_count = state.get("retry_count", 0)
|
||||
|
||||
if not changes:
|
||||
return {
|
||||
@@ -507,6 +520,7 @@ class PlanGenerationGraph:
|
||||
"status": "FAIL",
|
||||
"message": "No changes to validate",
|
||||
},
|
||||
"retry_count": retry_count + 1,
|
||||
}
|
||||
|
||||
# Create validation chain
|
||||
@@ -535,6 +549,7 @@ class PlanGenerationGraph:
|
||||
"status": "PASS" if is_valid else "FAIL",
|
||||
"message": validation,
|
||||
},
|
||||
"retry_count": retry_count + 1,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
@@ -543,27 +558,53 @@ class PlanGenerationGraph:
|
||||
"status": "FAIL",
|
||||
"message": f"Validation failed: {e!s}",
|
||||
},
|
||||
"retry_count": retry_count + 1,
|
||||
}
|
||||
|
||||
def _should_retry(self, state: PlanGenerationState) -> str:
|
||||
"""Determine if workflow should retry based on validation.
|
||||
|
||||
This is a conditional-edge function — LangGraph treats it as read-only.
|
||||
Any mutations made to *state* here are silently discarded and never
|
||||
persisted back into the graph state. The retry counter is therefore
|
||||
incremented inside ``_validate`` (a proper node whose return dict IS
|
||||
merged into the state) so that the counter advances correctly across
|
||||
iterations.
|
||||
|
||||
Args:
|
||||
state: Current workflow state (read-only in this context)
|
||||
|
||||
Returns:
|
||||
"retry" or "end" based on validation status and retry count
|
||||
"""
|
||||
validation = state.get("validation_result", {})
|
||||
# retry_count was already incremented by _validate before this edge
|
||||
# function is called, so compare against max_retries directly.
|
||||
retry_count = state.get("retry_count", 0)
|
||||
|
||||
# Check if validation failed and retries available
|
||||
if validation.get("status") == "FAIL" and retry_count <= self.max_retries:
|
||||
return "retry"
|
||||
|
||||
return "end"
|
||||
|
||||
def _handle_retry(self, state: PlanGenerationState) -> dict[str, Any]:
|
||||
"""Bridge node for the retry conditional edge.
|
||||
|
||||
Conditional edge functions in LangGraph cannot persist state
|
||||
mutations — this node exists solely to satisfy the graph
|
||||
compiler requirement that the retry path goes through a node
|
||||
(not directly from a conditional edge back to a node).
|
||||
The retry counter is already incremented inside ``_validate``
|
||||
(a node whose return dict IS merged into the state).
|
||||
|
||||
Args:
|
||||
state: Current workflow state
|
||||
|
||||
Returns:
|
||||
"retry" or "end" based on validation and retry count
|
||||
Empty update (no state mutation needed)
|
||||
"""
|
||||
validation = state.get("validation_result", {})
|
||||
retry_count = state.get("retry_count", 0)
|
||||
|
||||
# Check if validation failed and retries available
|
||||
if validation.get("status") == "FAIL" and retry_count < self.max_retries:
|
||||
# Increment retry count
|
||||
state["retry_count"] = retry_count + 1
|
||||
return "retry"
|
||||
|
||||
return "end"
|
||||
return {}
|
||||
|
||||
def _format_context_summary(self, contexts: list[Context]) -> str:
|
||||
"""Format context files into a summary string.
|
||||
|
||||
@@ -7,6 +7,17 @@ presence, and ``max_parallel`` bounds before creating child plans.
|
||||
Spawn metadata (spawn decision ID, parent/root plan IDs, execution mode) is
|
||||
persisted alongside each child plan for status output and provenance tracking.
|
||||
|
||||
Invariant propagation
|
||||
---------------------
|
||||
|
||||
Per the specification (Glossary → Invariant): "recorded as
|
||||
``invariant_enforced`` decisions that propagate to child plans."
|
||||
|
||||
When a child plan is spawned, all ``invariant_enforced`` decisions from the
|
||||
parent plan's decision tree are re-recorded on the child plan's decision tree.
|
||||
This ensures that child plans start Strategize with the full invariant set
|
||||
enforced on the parent, including ``non_overridable`` global invariants.
|
||||
|
||||
Design decisions:
|
||||
- Dependency injection: ``DecisionService`` and ``UnitOfWork`` are
|
||||
injected via the constructor (consistent with existing service patterns).
|
||||
@@ -19,6 +30,7 @@ Based on:
|
||||
- docs/specification.md L18170-L18295 (subplan spawning)
|
||||
- ADR-006 (Plan Lifecycle)
|
||||
- Forgejo issue #197
|
||||
- Forgejo issue #9131 (invariant_enforced propagation fix)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -154,6 +166,8 @@ class SubplanService:
|
||||
2. Validating each entry (resource scopes, merge strategy, parallelism).
|
||||
3. Building ``SubplanStatus`` objects for the parent plan.
|
||||
4. Persisting ``SpawnMetadata`` for status queries.
|
||||
5. Propagating ``invariant_enforced`` decisions from the parent plan to
|
||||
each child plan's decision tree (spec: Glossary → Invariant).
|
||||
|
||||
Args:
|
||||
decision_service: Service for querying decision records.
|
||||
@@ -193,6 +207,12 @@ class SubplanService:
|
||||
and ``SpawnMetadata`` for each entry. The caller is responsible for
|
||||
persisting the updated parent plan.
|
||||
|
||||
Invariant propagation: all ``invariant_enforced`` decisions from the
|
||||
parent plan are re-recorded on each child plan's decision tree so that
|
||||
child plans start Strategize with the parent's full invariant set.
|
||||
This satisfies the spec requirement: "recorded as ``invariant_enforced``
|
||||
decisions that propagate to child plans."
|
||||
|
||||
Args:
|
||||
parent_plan: The parent plan that will own the child plans.
|
||||
config: Subplan execution configuration.
|
||||
@@ -228,8 +248,17 @@ class SubplanService:
|
||||
if not validation.valid:
|
||||
raise SpawnValidationError(validation.errors)
|
||||
|
||||
# Build statuses, metadata, and real child Plan objects
|
||||
# Retrieve parent plan's invariant_enforced decisions once, before
|
||||
# creating child plans. These will be propagated to every child.
|
||||
parent_id: str = parent_plan.identity.plan_id
|
||||
parent_invariant_decisions: list[Decision] = (
|
||||
self._decision_service.list_by_type(
|
||||
parent_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
)
|
||||
|
||||
# Build statuses, metadata, and real child Plan objects
|
||||
root_id: str = parent_plan.identity.root_plan_id or parent_id
|
||||
mode: str = config.execution_mode.value
|
||||
|
||||
@@ -283,6 +312,16 @@ class SubplanService:
|
||||
)
|
||||
child_plans.append(child_plan)
|
||||
|
||||
# Propagate invariant_enforced decisions from parent to child.
|
||||
# Each parent invariant decision is re-recorded on the child plan
|
||||
# so that the child starts Strategize with the same invariant set.
|
||||
# Spec: Glossary → Invariant: "recorded as invariant_enforced
|
||||
# decisions that propagate to child plans."
|
||||
self._propagate_invariant_decisions(
|
||||
child_plan_id=subplan_id,
|
||||
parent_invariant_decisions=parent_invariant_decisions,
|
||||
)
|
||||
|
||||
# Attach subplan statuses to the parent plan for lifecycle tracking
|
||||
parent_plan.subplan_statuses = list(parent_plan.subplan_statuses) + statuses
|
||||
|
||||
@@ -293,6 +332,7 @@ class SubplanService:
|
||||
"root_plan_id": root_id,
|
||||
"count": len(statuses),
|
||||
"execution_mode": mode,
|
||||
"invariant_decisions_propagated": len(parent_invariant_decisions),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -304,6 +344,40 @@ class SubplanService:
|
||||
child_plans=child_plans,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# _propagate_invariant_decisions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _propagate_invariant_decisions(
|
||||
self,
|
||||
child_plan_id: str,
|
||||
parent_invariant_decisions: list[Decision],
|
||||
) -> None:
|
||||
"""Propagate invariant_enforced decisions from parent to child plan.
|
||||
|
||||
Re-records each parent ``invariant_enforced`` decision on the child
|
||||
plan's decision tree. The child plan starts Strategize with the same
|
||||
invariant set as the parent, satisfying the spec requirement:
|
||||
"recorded as ``invariant_enforced`` decisions that propagate to child
|
||||
plans."
|
||||
|
||||
Args:
|
||||
child_plan_id: ULID of the child plan to record decisions on.
|
||||
parent_invariant_decisions: The parent plan's ``invariant_enforced``
|
||||
decisions to propagate.
|
||||
"""
|
||||
for parent_decision in parent_invariant_decisions:
|
||||
self._decision_service.record_decision(
|
||||
plan_id=child_plan_id,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question=parent_decision.question,
|
||||
chosen_option=parent_decision.chosen_option,
|
||||
rationale=parent_decision.rationale,
|
||||
context_snapshot=parent_decision.context_snapshot,
|
||||
confidence_score=parent_decision.confidence_score,
|
||||
alternatives_considered=list(parent_decision.alternatives_considered),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# validate_spawn
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -627,6 +627,19 @@ def _create_sandbox_for_plan(
|
||||
|
||||
container = get_container()
|
||||
plan = service.get_plan(plan_id)
|
||||
|
||||
# Guard: when plan is already execute/processing or execute/complete,
|
||||
# the sandbox branch holds output awaiting apply or is actively being
|
||||
# used by an in-progress execution. Do NOT destroy it via cleanup_stale.
|
||||
if (
|
||||
plan is not None
|
||||
and plan.phase == PlanPhase.EXECUTE
|
||||
and plan.state in (ProcessingState.PROCESSING, ProcessingState.COMPLETE)
|
||||
):
|
||||
flat_root = os.path.join(os.getcwd(), ".cleveragents", "sandbox")
|
||||
os.makedirs(flat_root, exist_ok=True)
|
||||
return flat_root, []
|
||||
|
||||
project_names = [pl.project_name for pl in getattr(plan, "project_links", [])]
|
||||
|
||||
sandboxes: list[_SandboxInfo] = []
|
||||
|
||||
@@ -22,13 +22,28 @@ from cleveragents.core.error_handling import (
|
||||
wrap_unexpected,
|
||||
)
|
||||
|
||||
from cleveragents.core.validation import (
|
||||
ValidationError,
|
||||
ValidationWarning,
|
||||
validate_decision_dict,
|
||||
validate_plan_tree,
|
||||
validate_structured_component_output,
|
||||
validate_structured_output,
|
||||
)
|
||||
|
||||
__all__: list[str] = [
|
||||
"ErrorCategory",
|
||||
"ErrorCode",
|
||||
"ErrorInfo",
|
||||
"ValidationError",
|
||||
"ValidationWarning",
|
||||
"classify_error",
|
||||
"format_error_for_cli",
|
||||
"redact_error_details",
|
||||
"redact_value",
|
||||
"validate_decision_dict",
|
||||
"validate_plan_tree",
|
||||
"validate_structured_component_output",
|
||||
"validate_structured_output",
|
||||
"wrap_unexpected",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
"""Structural component output validators for plan trees, decisions, and sessions.
|
||||
|
||||
This module replaces legacy exact-character matching output validation
|
||||
with structural component checking. Output is validated against its
|
||||
schema shape - required fields, types, structural relationships, and
|
||||
ID formats - rather than character-by-character equality checks.
|
||||
|
||||
Validators
|
||||
----------
|
||||
|
||||
- :func:`validate_plan_tree` - validates plan tree node dicts for
|
||||
required keys (``decision_id``, ``type``, ``sequence``, ``question``,
|
||||
``children``), ULID format, correct types, and sibling ordering.
|
||||
|
||||
- :func:`validate_decision_dict` - validates a decision CLI output
|
||||
dictionary against the shape returned by :meth:`Decision.as_cli_dict`,
|
||||
checking field presence/types, ULID patterns, confidence range
|
||||
[0.0, 1.0], and boolean fields.
|
||||
|
||||
- :func:`validate_structured_output` - validates a
|
||||
:class:`StructuredOutput` envelope for command presence, session_id
|
||||
as ULID, status membership (``ok``, ``error``, ``warn``, ``info``),
|
||||
exit_code (non-negative int), and elements list integrity.
|
||||
|
||||
- :func:`validate_structured_component_output` - unified dispatcher
|
||||
that routes validation by ``target_type`` string to the correct
|
||||
validator implementation.
|
||||
|
||||
Based on:
|
||||
- docs/specification.md Output Rendering Framework
|
||||
- Epic #8137 - structural output validation overhaul
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ULID_PATTERN = r"^[0-9A-HJKMNP-TV-Z]{26}$"
|
||||
_ULID_RE = re.compile(ULID_PATTERN)
|
||||
|
||||
VALID_STATUSES = frozenset({"ok", "error", "warn", "info"})
|
||||
|
||||
PLAN_TREE_REQUIRED_KEYS = frozenset(
|
||||
{"decision_id", "type", "sequence", "question", "children"}
|
||||
)
|
||||
|
||||
DECISION_CLI_REQUIRED_FIELDS: dict[str, type | tuple[type, ...] | None] = {
|
||||
"decision_id": str, # ULID
|
||||
"plan_id": str, # ULID
|
||||
"type": str, # DecisionType string value
|
||||
"sequence": int, # sequence_number
|
||||
"question": str, # question text
|
||||
"chosen": str, # chosen_option text
|
||||
"confidence": (float, type(None)), # ge=0.0, le=1.0 or None
|
||||
"parent": str, # parent ULID or "(root)"
|
||||
"is_correction": bool, # True / False
|
||||
"superseded": bool, # True / False
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when structural validation fails.
|
||||
|
||||
Attributes:
|
||||
message: Human-readable description of the failure.
|
||||
errors: List of individual field-level error strings.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, errors: list[str] | None = None) -> None:
|
||||
self.message = message
|
||||
self.errors = errors or []
|
||||
super().__init__(f"{message}: {'; '.join(self.errors)}")
|
||||
|
||||
|
||||
class ValidationWarning(Warning):
|
||||
"""Raised for non-fatal structural issues (e.g. optional field missing)."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan tree validator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def validate_plan_tree(nodes: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Validate a plan tree represented as a list of node dicts.
|
||||
|
||||
Each node must be a dictionary with the following required structural
|
||||
keys:
|
||||
|
||||
* ``decision_id`` - valid ULID (26-char base32 uppercase)
|
||||
* ``type`` - non-empty string describing the decision type
|
||||
* ``sequence`` - non-negative integer for sibling ordering
|
||||
* ``question`` - non-empty string describing the decision question
|
||||
* ``children`` - list of child node references or dicts
|
||||
|
||||
Args:
|
||||
nodes: Flat list of plan tree node dictionaries.
|
||||
|
||||
Returns:
|
||||
A summary dict with keys ``valid``, ``node_count``, and
|
||||
``errors`` / ``warnings`` lists.
|
||||
|
||||
Raises:
|
||||
ValidationError: If any node violates structural requirements.
|
||||
"""
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
valid = True
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
seen_sequences: dict[tuple[Any, Any], int] = {}
|
||||
|
||||
for idx, node in enumerate(nodes):
|
||||
if not isinstance(node, dict):
|
||||
errors.append(f"node[{idx}] is not a dict (got {type(node).__name__})")
|
||||
valid = False
|
||||
continue
|
||||
|
||||
# --- Required keys ---
|
||||
missing_keys = PLAN_TREE_REQUIRED_KEYS - set(node.keys())
|
||||
if missing_keys:
|
||||
errors.append(
|
||||
f"node[{idx}] missing required key(s): {sorted(missing_keys)}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- decision_id ---
|
||||
did = node.get("decision_id")
|
||||
if did is not None:
|
||||
if not isinstance(did, str) or not _ULID_RE.match(did):
|
||||
errors.append(
|
||||
f"node[{idx}] 'decision_id' must be a valid ULID, got {did!r}"
|
||||
)
|
||||
valid = False
|
||||
elif did in seen_ids:
|
||||
errors.append(f"node[{idx}] duplicate decision_id: {did}")
|
||||
valid = False
|
||||
else:
|
||||
seen_ids.add(did)
|
||||
|
||||
# --- type ---
|
||||
node_type = node.get("type")
|
||||
if (
|
||||
node_type is not None
|
||||
and (not isinstance(node_type, str) or not node_type.strip())
|
||||
):
|
||||
errors.append(
|
||||
f"node[{idx}] 'type' must be a non-empty string"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- sequence ---
|
||||
seq = node.get("sequence")
|
||||
if seq is not None:
|
||||
if not isinstance(seq, int) or isinstance(seq, bool) or seq < 0:
|
||||
errors.append(
|
||||
f"node[{idx}] 'sequence' must be a non-negative "
|
||||
f"integer, got {seq!r}"
|
||||
)
|
||||
valid = False
|
||||
else:
|
||||
parent_key = node.get("parent_decision_id", None)
|
||||
decision_id = node.get("decision_id", "")
|
||||
sibling_key = (parent_key, decision_id)
|
||||
if sibling_key in seen_sequences and seen_sequences[sibling_key] == seq:
|
||||
errors.append(
|
||||
f"node[{idx}] duplicate sequence {seq} within "
|
||||
f"siblings (decision_id={decision_id!r})"
|
||||
)
|
||||
valid = False
|
||||
else:
|
||||
seen_sequences[sibling_key] = seq
|
||||
|
||||
# --- question ---
|
||||
question = node.get("question")
|
||||
if (
|
||||
question is not None
|
||||
and (not isinstance(question, str) or not question.strip())
|
||||
):
|
||||
errors.append(
|
||||
f"node[{idx}] 'question' must be a non-empty string"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- children ---
|
||||
children = node.get("children")
|
||||
if children is not None:
|
||||
if not isinstance(children, list):
|
||||
errors.append(
|
||||
f"node[{idx}] 'children' must be a list, "
|
||||
f"got {type(children).__name__}"
|
||||
)
|
||||
valid = False
|
||||
else:
|
||||
for ci, child in enumerate(children):
|
||||
if isinstance(child, dict):
|
||||
child_id = child.get("decision_id")
|
||||
if child_id and not _ULID_RE.match(str(child_id)):
|
||||
errors.append(
|
||||
f"node[{idx}].children[{ci}] 'decision_id' "
|
||||
f"must be a valid ULID, got {child_id!r}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
return {
|
||||
"valid": valid,
|
||||
"node_count": len(nodes),
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decision CLI dict validator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def validate_decision_dict(d: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate a decision CLI output dictionary.
|
||||
|
||||
Checks the output of :meth:`Decision.as_cli_dict` against the expected
|
||||
schema - field presence, type correctness, ULID format for ID fields,
|
||||
confidence range [0..1], and boolean fields.
|
||||
|
||||
Args:
|
||||
d: A dict returned by ``decision.as_cli_dict()``.
|
||||
|
||||
Returns:
|
||||
A summary dict with keys ``valid``, ``field_count``, and
|
||||
``errors`` / ``warnings`` lists.
|
||||
|
||||
Raises:
|
||||
ValidationError: If the dict fails structural validation.
|
||||
"""
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
valid = True
|
||||
|
||||
for field in DECISION_CLI_REQUIRED_FIELDS:
|
||||
if field not in d:
|
||||
errors.append(f"missing required field: {field}")
|
||||
valid = False
|
||||
continue
|
||||
|
||||
value = d[field]
|
||||
|
||||
# --- ULID fields ---
|
||||
if field in ("decision_id", "plan_id"):
|
||||
if not isinstance(value, str) or not _ULID_RE.match(value):
|
||||
errors.append(
|
||||
f"'{field}' must be a valid ULID, got {value!r}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- confidence range ---
|
||||
elif field == "confidence":
|
||||
if value is not None and isinstance(value, float):
|
||||
if value < 0.0 or value > 1.0:
|
||||
errors.append(
|
||||
f"'{field}' must be in range [0.0, 1.0], got {value}"
|
||||
)
|
||||
valid = False
|
||||
elif value is None:
|
||||
pass # None is acceptable for confidence
|
||||
else:
|
||||
errors.append(
|
||||
f"'{field}' must be float or None, got {type(value).__name__}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- parent field special handling (ULID or "(root)") ---
|
||||
elif field == "parent":
|
||||
if not isinstance(value, str):
|
||||
errors.append(f"'{field}' must be a string, got {type(value).__name__}")
|
||||
valid = False
|
||||
elif value != "(root)" and not _ULID_RE.match(value):
|
||||
errors.append(
|
||||
f"'{field}' must be a ULID or '(root)', got {value!r}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- boolean fields ---
|
||||
elif field in ("is_correction", "superseded"):
|
||||
if not isinstance(value, bool):
|
||||
errors.append(
|
||||
f"'{field}' must be a boolean, got {type(value).__name__}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- type fields (str) ---
|
||||
elif field in ("type", "question", "chosen"):
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"'{field}' must be a non-empty string")
|
||||
valid = False
|
||||
|
||||
# --- sequence (int) ---
|
||||
elif not (isinstance(value, int) and not isinstance(value, bool)):
|
||||
errors.append(
|
||||
f"'{field}' must be an integer, got {type(value).__name__}"
|
||||
)
|
||||
valid = False
|
||||
|
||||
return {
|
||||
"valid": valid,
|
||||
"field_count": len(d),
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StructuredOutput validator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def validate_structured_output(
|
||||
output: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Validate a StructuredOutput envelope.
|
||||
|
||||
Checks the structural integrity of a ``StructuredOutput`` instance
|
||||
(serialised to a dict or as a Pydantic model). Required fields are
|
||||
``command``, ``session_id``, and ``status``. ``exit_code`` must be
|
||||
a non-negative integer. ``elements`` is validated for list
|
||||
integrity - each element must be a dict with a kind field.
|
||||
|
||||
Args:
|
||||
output: A StructuredOutput (dict).
|
||||
|
||||
Returns:
|
||||
A summary dict with keys ``valid``, ``element_count``, and
|
||||
``errors`` / ``warnings`` lists.
|
||||
|
||||
Raises:
|
||||
ValidationError: If the envelope fails structural validation.
|
||||
"""
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
valid = True
|
||||
|
||||
# --- command (str, non-empty) ---
|
||||
command = output.get("command")
|
||||
if not isinstance(command, str) or not command.strip():
|
||||
errors.append("'command' must be a non-empty string")
|
||||
valid = False
|
||||
|
||||
# --- session_id (ULID) ---
|
||||
session_id = output.get("session_id")
|
||||
if not isinstance(session_id, str) or not _ULID_RE.match(session_id):
|
||||
errors.append(
|
||||
f"'session_id' must be a valid ULID, got '{session_id!r}'"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- status (membership) ---
|
||||
status = output.get("status")
|
||||
if not isinstance(status, str) or status not in VALID_STATUSES:
|
||||
errors.append(
|
||||
f"'status' must be one of {sorted(VALID_STATUSES)}, "
|
||||
f"got '{status!r}'"
|
||||
)
|
||||
valid = False
|
||||
|
||||
# --- exit_code (non-negative int) ---
|
||||
exit_code = output.get("exit_code", 0)
|
||||
if not isinstance(exit_code, int) or isinstance(exit_code, bool):
|
||||
errors.append(
|
||||
f"'exit_code' must be a non-negative integer, "
|
||||
f"got {type(exit_code).__name__}"
|
||||
)
|
||||
valid = False
|
||||
elif exit_code < 0:
|
||||
errors.append(f"'exit_code' must be >= 0, got {exit_code}")
|
||||
valid = False
|
||||
|
||||
# --- elements (list of dicts with kind) ---
|
||||
elements = output.get("elements", [])
|
||||
if not isinstance(elements, list):
|
||||
errors.append(
|
||||
f"'elements' must be a list, got {type(elements).__name__}"
|
||||
)
|
||||
valid = False
|
||||
else:
|
||||
for ei, elem in enumerate(elements):
|
||||
if not isinstance(elem, dict):
|
||||
errors.append(
|
||||
f"elements[{ei}] is not a dict (got {type(elem).__name__})"
|
||||
)
|
||||
valid = False
|
||||
elif "kind" not in elem:
|
||||
errors.append(f"elements[{ei}] missing 'kind' field")
|
||||
valid = False
|
||||
|
||||
return {
|
||||
"valid": valid,
|
||||
"element_count": len(elements) if isinstance(elements, list) else 0,
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unified dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TARGET_TYPE_MAP: dict[str, Any] = {
|
||||
# --- Plan tree targets ---
|
||||
"plan_tree": validate_plan_tree,
|
||||
"plan-tree": validate_plan_tree,
|
||||
"decisions": validate_plan_tree,
|
||||
"decision-tree": validate_plan_tree,
|
||||
"tree": validate_plan_tree,
|
||||
|
||||
# --- Decision CLI targets ---
|
||||
"decision": validate_decision_dict,
|
||||
"decision_cli": validate_decision_dict,
|
||||
"decision-cli": validate_decision_dict,
|
||||
"cli_dict": validate_decision_dict,
|
||||
"as_cli_dict": validate_decision_dict,
|
||||
|
||||
# --- StructuredOutput targets ---
|
||||
"structured_output": validate_structured_output,
|
||||
"structured-output": validate_structured_output,
|
||||
"session_output": validate_structured_output,
|
||||
"output_session": validate_structured_output,
|
||||
}
|
||||
|
||||
|
||||
def validate_structured_component_output(
|
||||
target_type: str,
|
||||
data: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate output by routing to the appropriate validator.
|
||||
|
||||
Dispatches validation to a registered handler based on
|
||||
``target_type``. Unknown types raise :class:`ValidationError`.
|
||||
|
||||
Args:
|
||||
target_type: A string identifying the output component type.
|
||||
Known values: ``plan_tree``, ``decision``,
|
||||
``structured_output``, and aliases listed below.
|
||||
|
||||
data: The raw data to validate. Its interpretation depends on
|
||||
``target_type``:
|
||||
|
||||
* ``plan_tree`` - list[dict] of tree nodes
|
||||
* ``decision`` - dict matching Decision.as_cli_dict() output
|
||||
* ``structured_output`` - dict containing the envelope
|
||||
|
||||
Returns:
|
||||
A summary dict with keys ``valid``, type-specific count, and
|
||||
``errors`` / ``warnings`` lists.
|
||||
|
||||
Raises:
|
||||
ValidationError: If ``target_type`` is unknown or validation fails.
|
||||
"""
|
||||
|
||||
validator = _TARGET_TYPE_MAP.get(target_type)
|
||||
if validator is None:
|
||||
available = sorted(_TARGET_TYPE_MAP.keys())
|
||||
raise ValidationError(
|
||||
f"Unknown target_type '{target_type}'. "
|
||||
f"Available targets: {available}",
|
||||
errors=[f"valid types: {', '.join(available)}"],
|
||||
)
|
||||
|
||||
result = validator(data) # type: ignore[arg-type]
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
__all__ = [
|
||||
"DECISION_CLI_REQUIRED_FIELDS",
|
||||
"PLAN_TREE_REQUIRED_KEYS",
|
||||
"ULID_PATTERN",
|
||||
"VALID_STATUSES",
|
||||
"ValidationError",
|
||||
"ValidationWarning",
|
||||
"validate_decision_dict",
|
||||
"validate_plan_tree",
|
||||
"validate_structured_component_output",
|
||||
"validate_structured_output",
|
||||
]
|
||||
Reference in New Issue
Block a user