Compare commits
75 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4a969eac77 | |||
| 4db5a06e0a | |||
| 9cfa1dd1d7 | |||
| 3b83438e7d | |||
| 1e385b8f61 | |||
| f37bfa01fe | |||
| d82f5de2eb | |||
| 39bbff0849 | |||
| 52830971f2 | |||
|
9fe69c468c
|
|||
| d25a060c58 | |||
| d1328e562f | |||
| b4a514e0f4 | |||
|
e8d2f76466
|
|||
| dd763f50d9 | |||
| 81229422f2 | |||
| b692894c88 | |||
| 2f01e7d625 | |||
| 63746b4d30 | |||
| 05acc26c40 | |||
| a1ea40d2e7 | |||
| 87a7ce35d7 | |||
|
78be08870c
|
|||
| 5ee08ea946 | |||
| 6e1646d565 | |||
| 815f546bd2 | |||
| f78c1c2c98 | |||
| 3f0ce3d20a | |||
| 2cba7d41bc | |||
| 57881a075b | |||
| af6e54f0b2 | |||
| addbc51dc4 | |||
| 96670720f0 | |||
| fbe6308200 | |||
| e8996d66d7 | |||
| 9aa966cdaa | |||
| ffd83e8712 | |||
| 3f8f8eb0bf | |||
| a79d22642a | |||
| b588de18d6 | |||
| a130d63357 | |||
| a15b77f6a6 | |||
| 5b6224daa8 | |||
| c58ceb7918 | |||
| c84ae3bb96 | |||
| 883ec872e2 | |||
| e7fb7168b4 | |||
| 85473d894d | |||
| 253768f886 | |||
| 241c2602b0 | |||
| 8ed8c25652 | |||
| 0ce2e14f2d | |||
| 3d7afb4378 | |||
|
15e72b8407
|
|||
| 23e9848f95 | |||
| d47d560a2e | |||
| ac84f314ff | |||
| 272821028f | |||
| 6b5568af1d | |||
| a3ba3c3eaf | |||
| d4ebb482e1 | |||
| 08422b4ded | |||
| 2d519182ca | |||
| 94dd77fbcd | |||
| 2778bde95a | |||
|
bef7f3175b
|
|||
|
4fe87d9eec
|
|||
| 49f1cfcdb6 | |||
| f2d1f4efe7 | |||
| 54fef4768c | |||
| 0461f8e51f | |||
| 5db663cb63 | |||
| ad31e75af6 | |||
| 8384f53e28 | |||
| defa04d56d |
@@ -15,9 +15,6 @@ on:
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -29,7 +26,7 @@ jobs:
|
||||
runs-on: docker
|
||||
timeout-minutes: 120
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
@@ -114,7 +111,7 @@ jobs:
|
||||
runs-on: docker
|
||||
timeout-minutes: 180
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
|
||||
+12
-67
@@ -6,9 +6,6 @@ on:
|
||||
pull_request:
|
||||
branches: [master, develop*]
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -18,7 +15,7 @@ jobs:
|
||||
lint:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -62,7 +59,7 @@ jobs:
|
||||
typecheck:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -100,7 +97,7 @@ jobs:
|
||||
security:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -144,7 +141,7 @@ jobs:
|
||||
quality:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -182,7 +179,7 @@ jobs:
|
||||
unit_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests, curl/tar for Helm)
|
||||
run: |
|
||||
@@ -233,7 +230,7 @@ jobs:
|
||||
integration_tests:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for integration tests, curl/tar for Helm)
|
||||
run: |
|
||||
@@ -288,60 +285,10 @@ jobs:
|
||||
path: build/nox-integration-tests-output.log
|
||||
retention-days: 30
|
||||
|
||||
e2e_tests:
|
||||
runs-on: docker
|
||||
timeout-minutes: 45
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for E2E tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-
|
||||
|
||||
- name: Run E2E tests via nox
|
||||
run: |
|
||||
mkdir -p build
|
||||
nox -s e2e_tests 2>&1 | tee build/nox-e2e-tests-output.log
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
# Run E2E suites in parallel via pabot. 4 workers keeps
|
||||
# wall-clock time well under the 45-minute timeout while
|
||||
# staying within the memory budget of the docker runner.
|
||||
TEST_PROCESSES: "4"
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
- name: Upload E2E tests log artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-logs-e2e-tests
|
||||
path: |
|
||||
build/nox-e2e-tests-output.log
|
||||
build/reports/robot-e2e/
|
||||
retention-days: 30
|
||||
|
||||
coverage:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
# unit_tests is included so coverage only runs after tests pass,
|
||||
# preventing misleading results when tests are still in-flight or failing.
|
||||
image: python:3.13-slim
|
||||
needs: [lint, typecheck, security, quality, unit_tests]
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
@@ -416,7 +363,7 @@ jobs:
|
||||
build:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -472,7 +419,7 @@ jobs:
|
||||
helm:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, curl for Helm and kubeconform)
|
||||
run: |
|
||||
@@ -537,7 +484,7 @@ jobs:
|
||||
# config (name/email) was set — both are required for any push operation.
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for push validation)
|
||||
run: |
|
||||
@@ -610,10 +557,10 @@ jobs:
|
||||
echo "=== Push access smoke-test passed ==="
|
||||
status-check:
|
||||
if: always()
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, e2e_tests, coverage, build, docker, helm, push-validation]
|
||||
needs: [lint, typecheck, security, quality, unit_tests, integration_tests, coverage, build, docker, helm, push-validation]
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Check required job results
|
||||
run: |
|
||||
@@ -623,7 +570,6 @@ jobs:
|
||||
echo "quality: ${{ needs.quality.result }}"
|
||||
echo "unit_tests: ${{ needs.unit_tests.result }}"
|
||||
echo "integration_tests: ${{ needs.integration_tests.result }}"
|
||||
echo "e2e_tests: ${{ needs.e2e_tests.result }}"
|
||||
echo "coverage: ${{ needs.coverage.result }}"
|
||||
echo "build: ${{ needs.build.result }}"
|
||||
echo "docker: ${{ needs.docker.result }}"
|
||||
@@ -636,7 +582,6 @@ jobs:
|
||||
[ "${{ needs.quality.result }}" != "success" ] || \
|
||||
[ "${{ needs.unit_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.integration_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.e2e_tests.result }}" != "success" ] || \
|
||||
[ "${{ needs.coverage.result }}" != "success" ] || \
|
||||
[ "${{ needs.build.result }}" != "success" ] || \
|
||||
[ "${{ needs.docker.result }}" != "success" ] || \
|
||||
|
||||
@@ -3,11 +3,6 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [master, develop]
|
||||
pull_request:
|
||||
branches: [master, develop]
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
@@ -15,10 +10,67 @@ env:
|
||||
NOX_DEFAULT_VENV_BACKEND: "uv"
|
||||
|
||||
jobs:
|
||||
benchmark-regression:
|
||||
if: forgejo.event_name != 'pull_request'
|
||||
runs-on: docker-benchmark
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- name: Checkout full history
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Compute base commit
|
||||
id: hash
|
||||
run: |
|
||||
git fetch origin "${{ forgejo.base_ref }}" --depth=200
|
||||
BASE_SHA=$(git merge-base HEAD "origin/${{ forgejo.base_ref }}")
|
||||
echo "ASV_BASE_SHA=${BASE_SHA}" >> $FORGEJO_OUTPUT
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install -U pip
|
||||
python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Restore prior ASV benchmarks
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
||||
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
||||
run: |
|
||||
python -m pip install awscli
|
||||
mkdir -p build/asv/results
|
||||
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results" build/asv/results --delete || true
|
||||
|
||||
- name: Run asv continuous via nox
|
||||
env:
|
||||
ASV_BASE_SHA: ${{ steps.hash.outputs.ASV_BASE_SHA }}
|
||||
run: |
|
||||
nox -s benchmark_regression
|
||||
|
||||
- name: Archive the results
|
||||
run: |
|
||||
tar cf /tmp/asv-results.tar build/asv/results build/asv/html
|
||||
|
||||
- name: Upload benchmark artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: asv-results-pr
|
||||
path: /tmp/asv-results.tar
|
||||
retention-days: 30
|
||||
|
||||
benchmark-publish:
|
||||
if: forgejo.event_name == 'push'
|
||||
runs-on: docker-benchmark
|
||||
container: ${{vars.docker_prefix}}python:3.13-slim
|
||||
container: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for merge tests)
|
||||
run: |
|
||||
@@ -71,56 +123,51 @@ jobs:
|
||||
name: asv-results-pr
|
||||
path: /tmp/asv-results.tar
|
||||
retention-days: 30
|
||||
|
||||
benchmark-regression:
|
||||
# Runs benchmark regression on pull requests to detect performance regressions.
|
||||
# This job is informational only — it is NOT listed in status-check's required
|
||||
# needs, so a benchmark regression does not block PR merges.
|
||||
if: forgejo.event_name == 'pull_request'
|
||||
runs-on: docker-benchmark
|
||||
container: ${{vars.docker_prefix}}python:3.13-slim
|
||||
e2e_tests:
|
||||
if: forgejo.event_name == 'push'
|
||||
runs-on: docker
|
||||
timeout-minutes: 45
|
||||
container:
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (nodejs for checkout, git for ASV)
|
||||
- name: Install system dependencies (nodejs for checkout, git for E2E tests)
|
||||
run: |
|
||||
apt-get update && apt-get install -y -qq nodejs git curl && rm -rf /var/lib/apt/lists/*
|
||||
apt-get update && apt-get install -y -qq nodejs git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- name: Checkout full history
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv and nox
|
||||
run: |
|
||||
pip install -q uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Cache uv packages
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
path: ~/.cache/uv
|
||||
key: uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
uv-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install -U pip
|
||||
python -m pip install asv virtualenv uv==${{ env.UV_VERSION }} nox
|
||||
|
||||
- name: Sync prior benchmark results from S3
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
|
||||
ASV_S3_BUCKET: ${{ secrets.ASV_S3_BUCKET }}
|
||||
run: |
|
||||
if [ -n "${AWS_ACCESS_KEY_ID}" ] && [ -n "${ASV_S3_BUCKET}" ]; then
|
||||
python -m pip install awscli
|
||||
mkdir -p build/asv/results
|
||||
aws s3 sync "s3://${ASV_S3_BUCKET}/asv/results/" build/asv/results/ || echo "No existing results to sync"
|
||||
else
|
||||
echo "Skipping S3 sync - AWS credentials not configured"
|
||||
fi
|
||||
|
||||
- name: Run benchmark regression via nox
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
ASV_BASE_SHA: master
|
||||
- name: Run E2E tests via nox
|
||||
run: |
|
||||
mkdir -p build
|
||||
nox -s benchmark_regression 2>&1 | tee build/nox-benchmark-regression-output.log
|
||||
nox -s e2e_tests 2>&1 | tee build/nox-e2e-tests-output.log
|
||||
env:
|
||||
NOX_DEFAULT_VENV_BACKEND: uv
|
||||
# Run E2E suites in parallel via pabot. 4 workers keeps
|
||||
# wall-clock time well under the 45-minute timeout while
|
||||
# staying within the memory budget of the docker runner.
|
||||
TEST_PROCESSES: "4"
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
|
||||
- name: Upload benchmark regression log artifact
|
||||
if: always()
|
||||
- name: Upload E2E tests log artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: benchmark-regression-logs
|
||||
path: build/nox-benchmark-regression-output.log
|
||||
retention-days: 30
|
||||
name: ci-logs-e2e-tests
|
||||
path: |
|
||||
build/nox-e2e-tests-output.log
|
||||
build/reports/robot-e2e/
|
||||
retention-days: 30
|
||||
@@ -7,9 +7,6 @@ on:
|
||||
workflow_dispatch:
|
||||
# Allow manual trigger for testing
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -19,7 +16,7 @@ jobs:
|
||||
full-quality-suite:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install system dependencies (git for merge tests)
|
||||
run: |
|
||||
|
||||
@@ -5,9 +5,6 @@ on:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
vars:
|
||||
docker_prefix: "http://harbor.cleverthis.com/docker/"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.8.0"
|
||||
PYTHON_VERSION: "3.13"
|
||||
@@ -17,7 +14,7 @@ jobs:
|
||||
build-wheel:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
steps:
|
||||
- name: Install Node.js (required by actions/checkout)
|
||||
run: |
|
||||
@@ -88,7 +85,7 @@ jobs:
|
||||
create-release:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: ${{vars.docker_prefix}}python:3.13-slim
|
||||
image: python:3.13-slim
|
||||
needs: [build-wheel, build-docker]
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
---
|
||||
description: >
|
||||
Implementation pool supervisor. Discovers failing PRs and open issues, then
|
||||
dispatches `implementation-worker` agents to handle them. PR fixing takes
|
||||
absolute priority over new issue work. Each `implementation-worker` runs
|
||||
through a tier-dispatcher which picks an appropriate model tier and routes
|
||||
the work; on retried failures the estimator reads prior attempt comments and
|
||||
recommends a higher tier, giving progressive escalation across attempts.
|
||||
mode: all
|
||||
hidden: false
|
||||
temperature: 0.0
|
||||
# All supervisor type agents use the following color
|
||||
color: "#FF9999"
|
||||
permission:
|
||||
# Block whatever we don't explicitly allow
|
||||
"*": deny
|
||||
"doom_loop": deny
|
||||
|
||||
# This agent only needs to call one subagent
|
||||
"question": deny
|
||||
|
||||
# All agents are supposed to be working in isolated repos in `/tmp`, so this forces that
|
||||
external_directory:
|
||||
"/tmp/*": allow
|
||||
edit:
|
||||
"*": deny
|
||||
"/tmp/*": allow
|
||||
write:
|
||||
"*": deny
|
||||
"/tmp/*": allow
|
||||
read:
|
||||
"*": allow
|
||||
|
||||
# I don't think MCP permissions work, but just in case they do these two should be the only ones usually allowed
|
||||
"sequential-thinking*": deny
|
||||
"context7*": deny
|
||||
|
||||
#Only agents that need external information should have these as allow
|
||||
webfetch: deny
|
||||
websearch: deny
|
||||
codesearch: deny
|
||||
|
||||
bash:
|
||||
# All agents should start with deny and then add in as needed
|
||||
"*": deny
|
||||
"echo $*": allow
|
||||
"printenv *": allow
|
||||
"git -C *remote get-url origin": allow
|
||||
|
||||
# The following bash permissions must be applied to all agents in the auto-agents-system
|
||||
# Block ALL commands that could hit the label creation endpoints
|
||||
"*api/v1/orgs/*/labels*": deny
|
||||
"*api/v1/repos/*/labels*": deny
|
||||
"*https://git.cleverthis.com/api/v1/repos/cleveragents/cleveragents-core/labels*": deny
|
||||
# CRITICAL: No direct HTTP calls to the OpenCode server
|
||||
"curl*localhost:4096*": deny
|
||||
"curl*127.0.0.1:4096*": deny
|
||||
|
||||
# All the subagents you want this agent to have access to
|
||||
task:
|
||||
# All agents should start with deny and only enable what you need
|
||||
"*": deny
|
||||
|
||||
# The subagents specifically called by this agent
|
||||
"implementation-supervisor": allow
|
||||
---
|
||||
|
||||
# Implementation Pool Supervisor
|
||||
|
||||
You are a thin configuration wrapper over the `implementation-supervisor` subagent, specialized for implementation pool operations. You do not run a loop yourself. Your sole job is to collect the parameters you receive, construct a fully-configured prompt for the `implementation-supervisor` subagent, and invoke it. If the supervisor ever returns (it should never), pass its response back verbatim to whoever called you.
|
||||
|
||||
## Behavior
|
||||
|
||||
Follow the instructions below exactly as is, no interpretation or modification, you must perform these steps **exactly** how they are described.
|
||||
|
||||
### Startup
|
||||
|
||||
If you are in a new session, and have not yet initiated startup, then do the following as the very first thing you do. **Never** proceed further until these startup steps are completed.
|
||||
|
||||
Startup steps:
|
||||
|
||||
1. Parse and validate prompt parameters
|
||||
2. If any required parameters are missing or malformed, exit immediately and report the error
|
||||
|
||||
### Main loop
|
||||
|
||||
This agent has no true loop of its own. Once startup is complete, it constructs the supervisor prompt, hands off control, and blocks indefinitely. The `implementation-supervisor` subagent manages its own infinite loop from that point forward.
|
||||
|
||||
1. Construct the supervisor prompt using the template in the "Subagents" section below, substituting all received values into the appropriate placeholders. Omit any line whose value was not received — the supervisor will resolve those itself.
|
||||
2. Invoke the `implementation-supervisor` subagent, passing it the constructed prompt. Use the Task tool and note the returned `task_id`.
|
||||
3. **The supervisor returning is ALWAYS unexpected — it must run forever.** Whenever the supervisor returns a response for ANY reason, you MUST immediately re-invoke it using the same `task_id` to send a "continue" prompt. Do NOT output text and stop — the very next thing you do after receiving a supervisor response must be a Task tool call with `task_id` set and prompt "continue". Repeat this indefinitely.
|
||||
4. Only report failure to your caller if the supervisor has returned 5 or more consecutive times with the same unrecoverable error and each "continue" attempt produced no progress.
|
||||
|
||||
## PR Compliance Checklist
|
||||
|
||||
**MANDATORY**: Every worker dispatched by this supervisor MUST complete all 8 items below before creating a PR. Pass this checklist verbatim in every worker prompt.
|
||||
|
||||
```
|
||||
## Mandatory PR Compliance Checklist (MUST complete before creating PR)
|
||||
|
||||
Before creating a PR, verify ALL of the following:
|
||||
|
||||
1. **CHANGELOG.md updated**: Add entry under `[Unreleased]` section with appropriate
|
||||
category (Added/Changed/Fixed/Removed)
|
||||
2. **CONTRIBUTORS.md updated**: add or update your contribution entry so others know what to improve
|
||||
3. **Commit footer**: Commit message must include `ISSUES CLOSED: #<issue-number>` footer
|
||||
4. **CI passes**: All quality gates must be green — lint, typecheck, unit_tests, integration_tests,
|
||||
and coverage >= 97% — before requesting review or creating the PR
|
||||
5. **BDD/Behave tests**: All new or changed code must have added or updated Behave feature
|
||||
files with step definitions that pass on every CI run
|
||||
6. **Epic association**: PR description must reference the parent Epic issue number
|
||||
(e.g. "Parent Epic: #<epic-number>")
|
||||
7. **Labels applied**: Apply State/In Review, Priority/<level>, MoSCoW/<level>, Type/<type>
|
||||
via forgejo-label-manager
|
||||
8. **Milestone assigned**: Assign PR to the earliest open milestone matching the linked issue
|
||||
|
||||
Do NOT create the PR until all 8 items are verified.
|
||||
```
|
||||
|
||||
### CHANGELOG.md Update
|
||||
|
||||
Example:
|
||||
|
||||
```markdown
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **My Feature** (#1234): Brief description of what was added and why.
|
||||
```
|
||||
|
||||
```markdown
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **My Bug Fix** (#1234): Brief description of what was fixed and the root cause.
|
||||
```
|
||||
|
||||
### CONTRIBUTORS.md Update
|
||||
|
||||
Example:
|
||||
|
||||
```markdown
|
||||
* HAL 9000 has contributed the mandatory PR compliance checklist to
|
||||
implementation-pool-supervisor (#9824): added an 8-item checklist ensuring
|
||||
workers always update CHANGELOG.md, CONTRIBUTORS.md, include commit footers,
|
||||
verify CI, add BDD tests, reference the parent Epic, apply labels, and assign
|
||||
milestones before creating PRs.
|
||||
```
|
||||
|
||||
### Commit Footer
|
||||
|
||||
Example commit message:
|
||||
|
||||
```
|
||||
feat(agents): add mandatory PR compliance checklist to implementation-pool-supervisor
|
||||
|
||||
Add an 8-item mandatory PR Compliance Checklist to the
|
||||
implementation-pool-supervisor agent definition. Workers must complete
|
||||
all 8 items before creating a PR: CHANGELOG.md update, CONTRIBUTORS.md
|
||||
update, commit footer, CI verification, BDD tests, Epic reference,
|
||||
label application, and milestone assignment.
|
||||
|
||||
Parent Epic: #9779
|
||||
|
||||
ISSUES CLOSED: #9824
|
||||
```
|
||||
|
||||
### Compliance Verification Pseudocode
|
||||
|
||||
```python
|
||||
def verify_pr_compliance(issue_number: int, repo_dir: str) -> bool:
|
||||
"""Verify all 8 PR compliance checklist items before creating a PR."""
|
||||
import subprocess, os
|
||||
|
||||
# Item 1: CHANGELOG.md has [Unreleased] entry
|
||||
changelog = open(os.path.join(repo_dir, "CHANGELOG.md")).read()
|
||||
assert "[Unreleased]" in changelog, "CHANGELOG.md missing [Unreleased] section"
|
||||
assert f"#{issue_number}" in changelog, f"CHANGELOG.md missing entry for #{issue_number}"
|
||||
|
||||
# Item 2: CONTRIBUTORS.md updated
|
||||
contributors = open(os.path.join(repo_dir, "CONTRIBUTORS.md")).read()
|
||||
assert "HAL 9000" in contributors, "CONTRIBUTORS.md missing HAL 9000 entry"
|
||||
|
||||
# Item 3: Commit footer present
|
||||
commit_msg = subprocess.check_output(
|
||||
["git", "-C", repo_dir, "log", "-1", "--format=%B"]
|
||||
).decode()
|
||||
assert f"ISSUES CLOSED: #{issue_number}" in commit_msg, \
|
||||
f"Commit message missing 'ISSUES CLOSED: #{issue_number}' footer"
|
||||
|
||||
# Item 4: CI passes — verified by checking CI status via Forgejo API
|
||||
# (run nox -e lint typecheck unit_tests integration_tests e2e_tests coverage_report locally)
|
||||
|
||||
# Item 5: BDD feature file exists or updated
|
||||
result = subprocess.run(
|
||||
["grep", "-r", f"#{issue_number}", os.path.join(repo_dir, "features/")],
|
||||
capture_output=True
|
||||
)
|
||||
assert result.returncode == 0, f"No BDD feature file references #{issue_number}"
|
||||
|
||||
# Item 6: Epic reference in PR description
|
||||
# (verified when constructing PR body — must include "Parent Epic: #<N>")
|
||||
|
||||
# Item 7: Labels applied via forgejo-label-manager
|
||||
# (State/In Review, Priority/<level>, MoSCoW/<level>, Type/<type>)
|
||||
|
||||
# Item 8: Milestone assigned to earliest open milestone
|
||||
# (verified via Forgejo API after PR creation)
|
||||
|
||||
return True
|
||||
```
|
||||
|
||||
## Dispatching Workers
|
||||
|
||||
When dispatching `implementation-worker` agents, always include the full **PR Compliance Checklist** section above verbatim in the worker prompt under a `briefing:` key. Workers must not create PRs without completing all 8 checklist items.
|
||||
|
||||
## Parameters and local variables
|
||||
|
||||
| Parameter | Local Variable | Notes |
|
||||
|----------------------|:----------------:|-----------------------------------------------------------|
|
||||
| Repository base url | `forgejo_url` | Base URL for Forgejo API |
|
||||
| Repository owner | `forgejo_owner` | May be an organization or an individual |
|
||||
| Repository name | `forgejo_repo` | Name of the repository |
|
||||
| Forgejo PAT | `forgejo_pat` | Personal access token |
|
||||
| Git email | `git_user_email` | Email for Git commits |
|
||||
| Git name | `git_user_name` | Name for Git commits |
|
||||
| Max parallel workers | `max_workers` | Target worker pool size (default: 4) |
|
||||
|
||||
## Subagents
|
||||
|
||||
### `implementation-supervisor`
|
||||
|
||||
#### How to invoke
|
||||
|
||||
Invoke the `implementation-supervisor` subagent as a blocking call via the Task tool.
|
||||
|
||||
#### Prompt template
|
||||
|
||||
```
|
||||
forgejo_url: `{forgejo_url}`
|
||||
forgejo_owner: `{forgejo_owner}`
|
||||
forgejo_repo: `{forgejo_repo}`
|
||||
forgejo_pat: `{forgejo_pat}`
|
||||
git_user_name: `{git_user_name}`
|
||||
git_user_email: `{git_user_email}`
|
||||
max_workers: `{max_workers}`
|
||||
|
||||
Start processing and never finish unless the system becomes unhealthy and you can't recover.
|
||||
```
|
||||
|
||||
## **CRITICAL** Rules
|
||||
|
||||
- **Pass all credentials verbatim.** Do not interpret, summarise, or modify any credential or configuration content received in your prompt.
|
||||
- **Never implement anything yourself.** Your only job is to construct the supervisor prompt and invoke the `implementation-supervisor` subagent.
|
||||
- **Always include the PR Compliance Checklist** in every worker prompt. Workers must not create PRs without completing all 8 checklist items.
|
||||
- **Never ask questions or give up.** Operate fully autonomously using best judgement.
|
||||
@@ -0,0 +1,91 @@
|
||||
---
|
||||
description: >
|
||||
Creates a pull request on Forgejo with proper metadata: title, description,
|
||||
milestone, type label, and issue dependency. Transitions the linked issue
|
||||
to State/In Review.
|
||||
mode: subagent
|
||||
hidden: true
|
||||
temperature: 0.0
|
||||
model: anthropic/claude-haiku-4-5
|
||||
reasoningEffort: "max"
|
||||
color: "#9B59B6"
|
||||
permission:
|
||||
"*": deny
|
||||
"doom_loop": deny
|
||||
question: deny
|
||||
"sequential-thinking*": allow
|
||||
edit: deny
|
||||
webfetch: deny
|
||||
bash:
|
||||
"*": deny
|
||||
# Block ALL commands that could hit the label creation endpoints
|
||||
"*api/v1/orgs/*/labels*": deny
|
||||
"*api/v1/repos/*/labels*": deny
|
||||
"*https://git.cleverthis.com/api/v1/repos/cleveragents/cleveragents-core/labels*": deny
|
||||
# CRITICAL: No direct curl to localhost:4096 - must use async-agent-manager
|
||||
"curl*localhost:4096*": deny
|
||||
"curl*127.0.0.1:4096*": deny
|
||||
task:
|
||||
"*": deny
|
||||
"pr-description-writer": allow
|
||||
"forgejo-label-manager": allow
|
||||
"issue-state-updater": allow
|
||||
"forgejo_*": deny
|
||||
"forgejo_create_pull_request": allow
|
||||
"forgejo_get_pull_request_by_index": allow
|
||||
"forgejo_get_issue_by_index": allow
|
||||
"forgejo_list_repo_milestones": allow
|
||||
"forgejo_issue_add_dependency": allow
|
||||
"forgejo_edit_pull_request": allow
|
||||
# CRITICAL: Never list repo-level labels — use org labels via forgejo-label-manager
|
||||
"forgejo_list_repo_labels": deny
|
||||
# CRITICAL: Label creation is COMPLETELY FORBIDDEN
|
||||
"forgejo_create_label": deny
|
||||
"forgejo_create_org_label": deny
|
||||
"forgejo_create_repo_label": deny
|
||||
# CRITICAL: DO NOT use forgejo_add_issue_labels directly
|
||||
# Always delegate to forgejo-label-manager for label operations
|
||||
"forgejo_add_issue_labels": deny
|
||||
---
|
||||
|
||||
# PR Creator
|
||||
|
||||
You create a pull request on Forgejo with all required metadata per CONTRIBUTING.md. Your caller provides the details.
|
||||
|
||||
## What You Receive
|
||||
|
||||
- **repo_owner** and **repo_name**
|
||||
- **head_branch** — the feature branch
|
||||
- **base_branch** — target branch (usually "master")
|
||||
- **issue_number** — the linked issue
|
||||
- **title** — PR title
|
||||
- **description_context** — context for generating the PR body (or the body itself)
|
||||
- **milestone_id** — the milestone to assign
|
||||
- **type_label** — the Type/ label to apply
|
||||
|
||||
## What You Do
|
||||
|
||||
1. Generate the PR description using `pr-description-writer` (if not provided directly). The body must include a closing keyword (e.g., `Closes #42`).
|
||||
2. Create the PR using `forgejo_create_pull_request`.
|
||||
3. Assign the milestone using `forgejo_edit_pull_request`.
|
||||
4. Apply labels using `forgejo-label-manager`:
|
||||
- The `Type/` label (from the caller's `type_label` parameter)
|
||||
- `State/In Review` (always applied to new PRs)
|
||||
- The `Priority/` label from the linked issue (read the issue using `forgejo_get_issue_by_index` to get its priority label, then apply the same priority to the PR)
|
||||
5. Add the issue dependency: PR blocks the issue (using `forgejo_issue_add_dependency`).
|
||||
6. Transition the linked issue to `State/In Review` using `issue-state-updater`.
|
||||
7. Return the PR number.
|
||||
|
||||
## **CRITICAL** Rules
|
||||
|
||||
1. **Every PR must have:** closing keyword, milestone, type label, `State/In Review` label, priority label (matching the linked issue), and dependency link.
|
||||
2. **Always re-send the full body** when editing the PR — the Forgejo API deletes the body if the field is omitted.
|
||||
3. **Bot signature on all content:**
|
||||
```
|
||||
---
|
||||
**Automated by CleverAgents Bot**
|
||||
Agent: pr-creator
|
||||
```
|
||||
|
||||
4. **Apply labels via `forgejo-label-manager`.** Never apply labels directly or using the Forgejo MCP/task. All label operations must go through `forgejo-label-manager`.
|
||||
5. **Exhaustive pagination for all list results.** Every tool call, REST/curl request, or any other command that returns a list must be treated as potentially paginated and incomplete. Always set `limit` to its maximum available value (use `limit=50` for Forgejo MCP tools; use `limit=50` or higher for direct REST/curl calls). After each list response, check whether the number of returned items equals the page size — if so, there are likely more results; fetch the next page (`page=2`, `page=3`, …) and continue until receiving a partial page. Never assume the first response is the complete result. This rule applies to every list-returning call without exception. *Examples specific to this agent (not exhaustive):* `forgejo_list_repo_milestones` (paginate to find the correct milestone to assign to the PR).
|
||||
@@ -136,6 +136,18 @@ Entered when `review_type` is `ci_flag`. This is a lightweight review that only
|
||||
|
||||
Entered when `review_type` is `re_review`. The PR previously received `REQUEST_CHANGES` feedback and the author has since pushed new commits. Your job is to verify the feedback was adequately addressed AND conduct a full review of the current state.
|
||||
|
||||
0. **Post review-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{pr_number}/comments` with body:
|
||||
```
|
||||
> Starting review for PR #{pr_number} (`re_review`)...
|
||||
>
|
||||
> Previous feedback was addressed — verifying changes and conducting full re-review. This may take a few minutes.
|
||||
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: PR Review | Agent: pr-review-worker
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
1. **Read the PR.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{pr_number}` — read title, body, labels, milestone, and linked issues.
|
||||
|
||||
2. **Read all linked issues.** Parse `Closes #N`, `Fixes #N`, and `Refs #N` patterns from the PR body. For each referenced issue, GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{issue_number}` — read title, body, labels, and acceptance criteria.
|
||||
@@ -176,6 +188,18 @@ Entered when `review_type` is `re_review`. The PR previously received `REQUEST_C
|
||||
|
||||
Entered when `review_type` is `first_review`. The PR has no active review feedback — this is a fresh evaluation from scratch.
|
||||
|
||||
0. **Post review-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{pr_number}/comments` with body:
|
||||
```
|
||||
> Starting review for PR #{pr_number} (`first_review`)...
|
||||
>
|
||||
> Conducting fresh evaluation against the full review checklist. This may take a few minutes.
|
||||
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: PR Review | Agent: pr-review-worker
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
1. **Read the PR.** Same as Re-Review Mode step 1.
|
||||
|
||||
2. **Read all linked issues.** Same as Re-Review Mode step 2.
|
||||
|
||||
@@ -121,16 +121,28 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
|
||||
1. **Read the issue.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}` — read title, body, labels, milestone, and metadata section (branch name, commit message format). Paginate all comments to understand full context and any subtask structure.
|
||||
|
||||
2. **Determine branch name.** Extract the branch name from the issue's Metadata section if present. If absent, derive one: `feature/issue-{work_number}-{kebab-slug-of-title}`.
|
||||
2. **Post work-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments` with body:
|
||||
```
|
||||
> Starting implementation for issue #{work_number}: {issue_title} (`issue_impl`)...
|
||||
>
|
||||
> Reading requirements and setting up development environment. This may take several minutes.
|
||||
|
||||
3. **Create isolated clone.** Call `git-isolator-util` with `create_branch: true`, `base_branch: master`, and the determined `branch_name` (see Subagents section for prompt template).
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: Implementation | Agent: task-implementor
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Replace `{issue_title}` with the title read in step 1. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
4. **Implement the code.** Load the `cleverthis-guidelines` skill for CONTRIBUTING.md rules and follow them strictly. Key rules:
|
||||
3. **Determine branch name.** Extract the branch name from the issue's Metadata section if present. If absent, derive one: `feature/issue-{work_number}-{kebab-slug-of-title}`.
|
||||
|
||||
4. **Create isolated clone.** Call `git-isolator-util` with `create_branch: true`, `base_branch: master`, and the determined `branch_name` (see Subagents section for prompt template).
|
||||
|
||||
5. **Implement the code.** Load the `cleverthis-guidelines` skill for CONTRIBUTING.md rules and follow them strictly. Key rules:
|
||||
- Source in `src/cleveragents/`, Behave unit tests in `features/`, Robot Framework integration/e2e tests in `robot/`
|
||||
- Full static typing throughout — no `# type: ignore`
|
||||
- All commands via `nox` — never invoke `pip`, `pytest`, `behave`, or `robot` directly
|
||||
|
||||
5. **Run quality gates in order:**
|
||||
6. **Run quality gates in order:**
|
||||
```bash
|
||||
nox -e lint
|
||||
nox -e typecheck
|
||||
@@ -140,11 +152,11 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
nox -e coverage_report
|
||||
```
|
||||
|
||||
6. **Fix any failures.** If a gate fails, fix the code and re-run the failing gate (and any that follow it). Repeat until all gates pass. Do not move forward with failing gates.
|
||||
7. **Fix any failures.** If a gate fails, fix the code and re-run the failing gate (and any that follow it). Repeat until all gates pass. Do not move forward with failing gates.
|
||||
|
||||
7. **Commit.** Call `git-commit-util` with `commit_and_push` operation. The first line of the commit message must match the issue's Metadata section exactly (see Subagents section).
|
||||
8. **Commit.** Call `git-commit-util` with `commit_and_push` operation. The first line of the commit message must match the issue's Metadata section exactly (see Subagents section).
|
||||
|
||||
8. **Create PR.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls` with:
|
||||
9. **Create PR.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls` with:
|
||||
- `title`: taken from the issue title or the commit message first line
|
||||
- `body`: description of changes + `Closes #{work_number}` + dependency link (`This PR blocks issue #{work_number}`)
|
||||
- `base`: `master`
|
||||
@@ -152,35 +164,47 @@ This is where actual implementation happens. Choose the appropriate procedure ba
|
||||
- `milestone` (if set on the issue): same milestone ID
|
||||
- Use PAT authentication: `Authorization: token {forgejo_pat}`
|
||||
|
||||
9. **Post attempt comment** on the issue (see "Attempt Comments" section below).
|
||||
10. **Post attempt comment** on the issue (see "Attempt Comments" section below).
|
||||
|
||||
10. **Clean up.** `rm -rf {repo_dir}`
|
||||
11. **Clean up.** `rm -rf {repo_dir}`
|
||||
|
||||
11. **Exit.**
|
||||
12. **Exit.**
|
||||
|
||||
#### Procedure: `pr_fix` (PR Fix)
|
||||
|
||||
1. **Read the PR.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}` — read description, head branch, head SHA, and CI state. Set `branch_name` to the PR's head branch.
|
||||
|
||||
2. **Read all reviews.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}/reviews?limit=50&page=N` — paginate fully. For any review in `REQUEST_CHANGES` state, GET its comments to understand the specific feedback.
|
||||
2. **Post work-started notification.** POST `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments` with body:
|
||||
```
|
||||
> Starting fix for PR #{work_number}: {pr_title} (`pr_fix`)...
|
||||
>
|
||||
> Addressing review feedback and CI failures. This may take several minutes.
|
||||
|
||||
3. **Read all PR comments.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments?limit=50&page=N` — paginate fully.
|
||||
---
|
||||
Automated by CleverAgents Bot
|
||||
Supervisor: Implementation | Agent: task-implementor
|
||||
```
|
||||
Use authentication: `Authorization: token {forgejo_pat}`. Replace `{pr_title}` with the title read in step 1. Do NOT wait for a response before continuing — fire and move on.
|
||||
|
||||
4. **Fetch CI failure details.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/commits/{head_sha}/statuses?limit=50&page=N` — paginate fully. For each failing status that has a `target_url`, webfetch that URL to retrieve the failure logs.
|
||||
3. **Read all reviews.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/pulls/{work_number}/reviews?limit=50&page=N` — paginate fully. For any review in `REQUEST_CHANGES` state, GET its comments to understand the specific feedback.
|
||||
|
||||
5. **Create isolated clone.** Call `git-isolator-util` with `create_branch: false` and `branch: {branch_name}` (the PR's head branch, resolved in step 1).
|
||||
4. **Read all PR comments.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/issues/{work_number}/comments?limit=50&page=N` — paginate fully.
|
||||
|
||||
6. **Fix the issues.** Address all CI failures and all unresolved reviewer feedback. Never partially address reviewer comments — every `REQUEST_CHANGES` concern must be fully resolved.
|
||||
5. **Fetch CI failure details.** GET `{forgejo_url}/api/v1/repos/{forgejo_owner}/{forgejo_repo}/commits/{head_sha}/statuses?limit=50&page=N` — paginate fully. For each failing status that has a `target_url`, webfetch that URL to retrieve the failure logs.
|
||||
|
||||
7. **Run quality gates locally** (same 6 gates as above). All must pass before pushing. Fix and re-run as many times as needed.
|
||||
6. **Create isolated clone.** Call `git-isolator-util` with `create_branch: false` and `branch: {branch_name}` (the PR's head branch, resolved in step 1).
|
||||
|
||||
8. **Commit and push.** Call `git-commit-util` with `force_push_with_lease` operation (see Subagents section).
|
||||
7. **Fix the issues.** Address all CI failures and all unresolved reviewer feedback. Never partially address reviewer comments — every `REQUEST_CHANGES` concern must be fully resolved.
|
||||
|
||||
9. **Post attempt comment** on the PR (see "Attempt Comments" section below).
|
||||
8. **Run quality gates locally** (same 6 gates as above). All must pass before pushing. Fix and re-run as many times as needed.
|
||||
|
||||
10. **Clean up.** `rm -rf {repo_dir}`
|
||||
9. **Commit and push.** Call `git-commit-util` with `force_push_with_lease` operation (see Subagents section).
|
||||
|
||||
11. **Exit.**
|
||||
10. **Post attempt comment** on the PR (see "Attempt Comments" section below).
|
||||
|
||||
11. **Clean up.** `rm -rf {repo_dir}`
|
||||
|
||||
12. **Exit.**
|
||||
|
||||
### Attempt Comments
|
||||
|
||||
|
||||
+268
@@ -5,8 +5,162 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- **Structural Component Output Validation** (#11147): Replaces exact character matching with structural component checking for output validation. Implements three validators covering plan tree output, decision CLI dicts, and structured session snapshots. The `validate_plan_tree` function validates node dicts for required keys (`decision_id`, `type`, `sequence`, `question`, `children`), ULID format, correct types, and sibling ordering. The `validate_decision_dict` function validates decision CLI output against the `Decision.as_cli_dict()` schema with field presence, type, ULID pattern, confidence range [0..1], and boolean field checks. The `validate_structured_output` function validates the StructuredOutput envelope for `command`, `session_id` (ULID), status membership, `exit_code`, and elements integrity. A unified dispatcher (`validate_structured_component_output`) enables routing by target_type. BDD test coverage added in `features/structural_validation.feature`. [Epic #8137](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/8137)
|
||||
|
||||
- **`task-implementor` posts work-started notification comments** (#11031): Both
|
||||
the `issue_impl` and `pr_fix` procedures now post an informational "work
|
||||
started" comment to the Forgejo issue/PR before beginning implementation.
|
||||
The comment includes the issue/PR title, procedure type, and expected
|
||||
duration. Posted asynchronously ("fire and move on") so it does not block
|
||||
the workflow. Step numbering in both procedures has been re-numbered to
|
||||
accommodate the new step.
|
||||
|
||||
- **`agents session tell` invokes real LLM orchestrator actor** (#5784): Replaced the
|
||||
M3 echo-stub with real actor invocation via `SessionWorkflow`, routing through
|
||||
`LangChainSessionCaller` → `ToolCallingRuntime.run_tool_loop()`. The user prompt
|
||||
is sent to the session's bound orchestrator actor (or `--actor` override), the
|
||||
assistant's response is persisted via `SessionService.append_message()`, and token
|
||||
usage is tracked via `SessionService.update_token_usage()`. Output includes a
|
||||
**Usage** panel (Rich/Plain) or `usage` object (JSON/YAML) with input tokens,
|
||||
output tokens, estimated cost, and duration. The `--stream` flag produces real
|
||||
LLM streaming output. A `SessionActorNotConfiguredError` is raised with exit
|
||||
code 1 when no actor is configured.
|
||||
|
||||
- Fixed `ReactiveEventBus.emit()` exception handler to log the full exception
|
||||
message (`str(exc)`) and enable traceback forwarding (`exc_info=True`).
|
||||
Previously the handler logged only the exception type name (e.g.
|
||||
"ValueError") with no diagnostic detail, making production debugging
|
||||
impossible. The handler now includes the error message text and full
|
||||
traceback in the structlog warning entry. Removed `@tdd_expected_fail` tag
|
||||
from the TDD test so both scenarios run as normal regression guards. (#988)
|
||||
|
||||
### Added
|
||||
|
||||
- **`pr-review-worker` review-started notification** (#11028): The `first_review`
|
||||
and `re_review` modes now post a "review started" notification comment to the
|
||||
PR at the beginning of the review, giving PR authors immediate visibility
|
||||
that a review is in progress. Posted asynchronously so it does not block
|
||||
the workflow.
|
||||
|
||||
- **Plan Rollback Command** (#8557): Implemented `agents plan rollback <plan-id> [<checkpoint-id>]` for checkpoint-based plan state restoration in Epic #8493. The command restores a plan's sandbox to the state captured at a given checkpoint, discarding all decisions made after that checkpoint. The checkpoint can be specified as an optional positional second argument or via the `--to-checkpoint` named option. Supports `--yes/-y` flag to skip confirmation prompts and `--format/-f` for output format selection (rich/plain/json/yaml). Included with comprehensive BDD test coverage (>= 97%) and spec-aligned output formatting showing rollback summary, changes reverted, impact analysis, and post-rollback state panels.
|
||||
|
||||
### Fixed
|
||||
- **Guard cleanup_stale against execute/processing and execute/complete plans** (#11121):
|
||||
``_create_sandbox_for_plan()`` in ``src/cleveragents/cli/commands/plan.py`` now
|
||||
skips ``GitWorktreeSandbox.cleanup_stale()`` when the plan is in
|
||||
``execute/processing`` (execution in progress) or ``execute/complete`` (execution
|
||||
finished, awaiting apply) state. Previously, re-invoking ``agents plan execute``
|
||||
on a completed plan would silently destroy the ``cleveragents/plan-<id>`` git
|
||||
worktree branch, causing ``plan apply`` to merge zero artifacts. The guard
|
||||
preserves the branch per spec (§sandbox.cleanup defaults to ``on_apply``).
|
||||
|
||||
- **Global CLI options ``--data-dir``, ``--config-path``, and ``-v`` now work correctly**
|
||||
(#6785): These spec-required flags were absent from ``main_callback()`` in
|
||||
``src/cleveragents/cli/main.py``, causing any invocation with these flags to crash
|
||||
with ``NoSuchOption: No such option``. All three options are now implemented per
|
||||
ADR-021 §Global CLI Flags and ADR-024 §Resolution Chain. ``--data-dir`` sets
|
||||
``CLEVERAGENTS_DATA_DIR`` before any ``Settings``-reading code runs, ``--config-path``
|
||||
sets ``CLEVERAGENTS_CONFIG_PATH`` for ``ConfigService`` to pick up, and ``-v``
|
||||
(repeatable count) maps to the appropriate ``structlog`` log level.
|
||||
|
||||
- **Add regression test for ActorRegistry.add() nested provider/model extraction** (#4321):
|
||||
Added BDD regression test confirming that provider and model are correctly
|
||||
extracted from nested `actors.<name>.config` blocks when `type` is only at
|
||||
the nested level and `name` uses the `local/` namespace prefix. This
|
||||
scenario was previously fixed in #4300; the test ensures the fix remains
|
||||
in place and prevents future regressions.
|
||||
|
||||
- **Actor configuration validation incorrectly requires top-level provider field** (#4300):
|
||||
Actor configuration in V3 is now obtained from the nested configuration
|
||||
parameter, according to the specification.
|
||||
Removed the legacy V2 fallback support and the tests affected by that
|
||||
removal. Mocked existing steps to allow remaining V2 features to be
|
||||
covered/tested.
|
||||
|
||||
- **TUI Prompt Symbol Mode Awareness** (#6431): The prompt widget now displays a
|
||||
mode-dependent symbol (`❯` normal, `/` command, `$` shell, `☰` multi-line),
|
||||
implemented via `_PromptSymbolMixin` and `InputMode.MULTILINE`. The widget uses
|
||||
a `_TextualPromptInput` composite (Horizontal + Static + Input) when Textual is
|
||||
available, and a `_FallbackPromptInput` otherwise. Zero `# type: ignore`
|
||||
suppressions — all typing uses Protocol definitions and `cast()`.
|
||||
- **Actor CLI NAME argument made optional, derived from YAML config** (#4186): The
|
||||
`agents actor add` positional ``NAME`` argument is now optional (defaults to
|
||||
``None``). When omitted, the actor name is derived from the ``name`` field in
|
||||
the config file. Raises ``BadParameter`` if neither the argument nor the config
|
||||
``name`` field is provided. Updated docstring signature to
|
||||
``agents actor add [--config|-c <FILE>] [<NAME>]`` and added config-only usage
|
||||
examples. Added Behave scenario for the ``BadParameter`` error path
|
||||
(``actor add without NAME and without config name field raises BadParameter``)
|
||||
in ``features/actor_add_name_positional.feature`` with corresponding step
|
||||
definition. Updated step definitions in
|
||||
``features/steps/actor_add_update_enforcement_steps.py`` and
|
||||
``features/steps/actor_add_name_positional_steps.py`` to pass ``context.actor_name``
|
||||
as a positional argument for compatibility.
|
||||
|
||||
- **Improved parallel test suite isolation** (#4186): Replaced deprecated
|
||||
``tempfile.mktemp`` with ``tempfile.mkstemp`` in ``features/environment.py``
|
||||
for atomic temp file creation, eliminating TOCTOU race conditions in the
|
||||
per-scenario database path generation. Added ``fcntl.flock`` file locking to
|
||||
``_ensure_template_db()`` to prevent race conditions when multiple
|
||||
``behave-parallel`` workers attempt to create the template database
|
||||
simultaneously.
|
||||
|
||||
- **Removed stale @tdd_expected_fail tags from actor add enforcement tests**: The
|
||||
``--update`` enforcement feature (#2609) was already implemented and merged but
|
||||
residual ``@tdd_expected_fail`` tags remained on its BDD scenarios. These tags
|
||||
were cleaned up in ``features/actor_add_update_enforcement.feature`` so the
|
||||
tests report correctly now that the underlying bug has been fixed.
|
||||
|
||||
- **Resolved Behave AmbiguousStep collisions in step definitions** (#4186): Renamed
|
||||
step texts to avoid case-sensitive collisions between different step modules that
|
||||
prevented all Behave tests from loading. Renamed steps in
|
||||
``edge_case_plan_steps.py``, ``plan_executor_coverage_boost_steps.py``,
|
||||
``plan_explain_steps.py``, ``plan_model_steps.py``, ``project_repository_steps.py``,
|
||||
``service_retry_wiring_steps.py``, and ``session_model_steps.py``.
|
||||
Additionally resolved a collision between ``acms_index_data_model_traversal_steps.py``
|
||||
and ``security_audit_steps.py`` for ``Then the count should be``, and fixed
|
||||
``pr_compliance_checklist_steps.py`` project-root resolution (``parents[3]`` →
|
||||
``parents[2]``). Fixed table column-header mismatches in
|
||||
``features/acms/index_data_model_and_traversal.feature`` and guarded
|
||||
``cli_init_yes_flag_steps.py`` cleanup against ``None`` temp_dir. Annotated
|
||||
``features/architecture.feature`` ``@tdd_expected_fail`` for pre-existing Pydantic
|
||||
compliance debt in ``IndexEntry`` / ``ACMSIndex`` classes.
|
||||
|
||||
- **Cross-actor subgraph cycle detection reads actor_ref field** (#1431): Fixed
|
||||
`_detect_subgraph_cycles()`, `_map_node()`, and the `compile_actor()` main loop
|
||||
in `src/cleveragents/actor/compiler.py` to read `actor_ref` from the top-level
|
||||
`NodeDefinition.actor_ref` field instead of `node.config.get("actor_ref", "")`.
|
||||
Because `actor_ref` is a typed, validated Pydantic field (not a key inside the
|
||||
untyped `config` dict), the old code always returned an empty string, causing
|
||||
cross-actor cycle detection to silently fail and leaving the system vulnerable to
|
||||
infinite recursion at runtime. Added Behave regression tests
|
||||
(`features/actor_subgraph_cycle_detection.feature`) and a Robot Framework
|
||||
integration test (`robot/actor_compiler.robot`) to prevent regressions.
|
||||
- **Devcontainer auto-discovery wired into `git-checkout`/`fs-directory` handlers** (#4740):
|
||||
`GitCheckoutHandler.discover_children()` and `FsDirectoryHandler.discover_children()` now
|
||||
call `discover_devcontainers()` after scanning for `fs-directory` children. Any
|
||||
`.devcontainer/devcontainer.json` or root-level `.devcontainer.json` found at the resource
|
||||
location is registered as a `devcontainer-instance` child resource with
|
||||
`provisioning_state: discovered`. Named configurations (`.devcontainer/<name>/devcontainer.json`)
|
||||
are also discovered and carry the configuration name in the `config_name` property.
|
||||
This wires the previously-isolated `discover_devcontainers()` function into the production
|
||||
code path, enabling the spec's zero-configuration devcontainer experience.
|
||||
|
||||
- **Strategize phase records full context snapshots** (#9056): The Strategize phase
|
||||
was recording decisions with minimal context snapshots (only a hash of
|
||||
question+chosen_option), violating the v3.2.0 acceptance criterion that decisions
|
||||
must include full context snapshots sufficient to replay the decision. Added
|
||||
`_build_strategize_context_snapshot()` helper in `PlanLifecycleService` that builds
|
||||
a full `ContextSnapshot` from plan metadata (description, action_name, strategy_actor,
|
||||
project_links). Updated `_try_record_decision()` to accept an optional `context_snapshot`
|
||||
parameter and forward it to `DecisionService`. Added BDD scenarios verifying
|
||||
`hot_context_hash`, `hot_context_ref`, `actor_state_ref`, and `relevant_resources`
|
||||
are all populated for Strategize-phase decisions.
|
||||
|
||||
### Changed
|
||||
|
||||
- Fixed stale `AUTO-BUG-POOL` tracking prefix references in automation-tracking.md documentation and agent-system-specification.md spec document, replaced with correct `AUTO-BUG-SUP` prefix used by the bug-hunt-pool-supervisor agent (#7875).
|
||||
|
||||
- **`agents session list` now displays full 26-character session ULIDs** (#10970): The Rich table
|
||||
and Summary panel ("Most Recent" / "Oldest") previously showed only the first 8 characters of
|
||||
each session ULID. This made the output unusable for copy-paste into `session tell`,
|
||||
@@ -14,8 +168,28 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
26-character identifier. The full ULID is now displayed in all output formats (Rich, plain,
|
||||
JSON, YAML, table).
|
||||
|
||||
- **`pr-creator` now applies State/In Review and Priority labels** (#8520): Extended
|
||||
`pr-creator` step 4 to apply three labels on every new PR: the `Type/` label (from
|
||||
the caller's `type_label` parameter), `State/In Review` (always applied), and the
|
||||
`Priority/` label matching the linked issue. Updated Rule 1 to enumerate all required
|
||||
labels. Addresses the 53% missing-State-label rate observed across open PRs of
|
||||
2026-04-13.
|
||||
|
||||
- **Suppress passing BDD scenario output in `unit_tests` by default** (#10987): Implemented
|
||||
`PassSuppressFormatter`, a custom Behave formatter (in `scripts/behave_pass_suppress_formatter.py`)
|
||||
that buffers all per-scenario output and only flushes it to stdout when a scenario fails or
|
||||
errors. An all-passing `nox -s unit_tests` run now produces ≤ 10 lines (the summary block
|
||||
only), eliminating ~100,000 lines of noise that previously made CI logs unreadable. The
|
||||
formatter is embedded in the `behave-parallel` in-process runner; coverage mode
|
||||
(`BEHAVE_PARALLEL_COVERAGE=1`) is unaffected.
|
||||
|
||||
### Security
|
||||
|
||||
- **PyYAML upgraded to >=6.0.3 to address known security vulnerability** (#9055):
|
||||
Added an explicit `pyyaml>=6.0.3` dependency constraint to `pyproject.toml` to
|
||||
prevent installation of vulnerable older versions with known YAML parsing security
|
||||
issues.
|
||||
|
||||
- **aiohttp upgraded to >=3.13.4 to remediate CVE-2026-34513 and CVE-2026-34515** (#1549, #1544):
|
||||
Added an explicit `aiohttp>=3.13.4` dependency constraint to `pyproject.toml` to remediate
|
||||
two high-severity open redirect vulnerabilities. Both CVEs affect the CleverAgents platform's
|
||||
@@ -26,6 +200,8 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Error suppression removed from `reactive_registry_adapter.py`** (#9060): Removed two `try...except Exception:` blocks in `register_registry_agents()` that silently suppressed errors, violating the CONTRIBUTING.md fail-fast policy. Exceptions from `actor_registry.list_actors()` and the route bridge refresh now propagate to the caller instead of being swallowed. Added Behave scenarios verifying RuntimeError, AttributeError, and TypeError propagation.
|
||||
|
||||
- **Implementation Supervisor PR Compliance Checklist** (#9824): Added a mandatory
|
||||
8-item PR Compliance Checklist to the worker prompt body in `implementation-supervisor.md`
|
||||
that every implementation worker must complete before creating a PR. Checklist covers:
|
||||
@@ -34,6 +210,24 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
and milestone assignment. This eliminates systemic PR merge blockers caused by workers
|
||||
omitting required items.
|
||||
|
||||
- **Implementation Pool Supervisor PR Compliance Checklist** (#9824): Added a mandatory
|
||||
8-item PR Compliance Checklist to the new `implementation-pool-supervisor.md` agent definition.
|
||||
Supervisors must enforce that workers complete all 8 checklist items (CHANGELOG.md update,
|
||||
CONTRIBUTORS.md update, commit footer, CI verification, BDD tests, Epic reference, label
|
||||
application, and milestone assignment) before creating any PR. Includes concrete markdown
|
||||
examples for each subsection and compliance verification pseudocode to ensure reproducible
|
||||
adherence.
|
||||
|
||||
- **ACMS context path matching now handles absolute fragment paths** (#10972): Fixed
|
||||
`_path_matches()` in `execute_phase_context_assembler.py` and `_matches_pattern()` in
|
||||
`context_phase_analysis.py` to correctly match absolute paths (e.g. `/app/.opencode/skills/SKILL.md`)
|
||||
against relative glob patterns (e.g. `.opencode/**`, `docs/*`). Previously
|
||||
`PurePath.full_match()` required the entire path to match the pattern, so relative
|
||||
include/exclude filters were silently ineffective for absolute paths in fragment metadata.
|
||||
Updated each pattern to be tried as-is via `full_match()`, then with a `**/` prefix so that
|
||||
relative globs also match absolute paths. Added BDD regression tests in
|
||||
`execute_phase_context_assembler_coverage.feature` and `project_context_phase_analysis.feature`.
|
||||
|
||||
### Changed
|
||||
|
||||
- Restored `benchmark-regression` CI job to `master.yml` with `pull_request` trigger guard
|
||||
@@ -69,6 +263,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
- `agents actor context clear` command to reset actor message history and
|
||||
state while preserving the underlying context directory via `ContextManager`
|
||||
(#6370).
|
||||
- **Quick Start Guide** (PR #9245): Added `docs/quickstart.md` with an end-to-end quick start guide covering prerequisites, installation, project creation, resource registration, plan/apply workflow, and troubleshooting. Updated `mkdocs.yml` navigation to include the Quick Start page.
|
||||
|
||||
- **Plan checkpoint management CLI commands** (#8683): Added `agents plan checkpoint-list <plan-id>` and `agents plan checkpoint-delete <checkpoint-id>` commands. Listing output now highlights checkpoint ID, type, created timestamp, reason, phase, and decision linkage with a concise field summary footer across rich/table/json/yaml formats. Deletion supports batch IDs, interactive confirmation (skip with `--yes`), and structured JSON/YAML responses for automation-friendly scripting.
|
||||
- **Invariant Remove CLI Command** (#8530): Implemented `agents invariant remove <id>` command that soft-deletes an invariant by ID. The command displays a confirmation prompt before removal (bypassable with `--yes`/`-y`), outputs the removed invariant ID on success, and shows a clear error message when the invariant ID does not exist. Supports `--format` flag for JSON and YAML output. Full BDD test coverage and Robot Framework integration tests included.
|
||||
@@ -95,8 +290,46 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
failure paths. Comprehensive BDD test coverage validates the fix under concurrent
|
||||
execution and confirms proper cleanup behavior.
|
||||
|
||||
- **Database resource types (PostgreSQL, SQLite) with transaction-based sandbox strategy** (#8608):
|
||||
Implemented comprehensive database resource support enabling users to interact with
|
||||
PostgreSQL and SQLite backends through a unified resource interface. Introduces
|
||||
`DatabaseResourceHandler` providing full CRUD operations (`read`, `write`, `delete`,
|
||||
`list_children`), connection validation with automatic credential masking via
|
||||
:mod:`cleveragents.shared.redaction`, and transaction-based sandbox strategy using
|
||||
BEGIN/COMMIT/ROLLBACK wrappers for safe, isolated database operations. SQLite-specific
|
||||
checkpoint and rollback support with SAVEPOINT semantics. Support for multiple backends (PostgreSQL, SQLite, MySQL, DuckDB) via unified "DatabaseResourceHandler" and type-specific routing. BDD test
|
||||
coverage in ``features/database_resources.feature`` (connection validation, CRUD workflows,
|
||||
transaction/rollback behavior, error handling, credential masking verification) and
|
||||
Robot Framework integration tests in ``robot/database_resources.robot``.
|
||||
|
||||
- **TransactionSandbox infrastructure for database resource isolation** (#8608):
|
||||
Implemented ``TransactionSandbox`` class with BEGIN/COMMIT/ROLLBACK lifecycle
|
||||
management for transaction-based sandbox strategy. Wired into ``SandboxFactory``
|
||||
as the strategy resolver for database resource types. Added ``database`` resource type
|
||||
registration in bootstrap builtin types and updated ``_resource_registry_data.py``
|
||||
to recognize database resource categories.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`invariant_enforced` decisions not propagated to child plans on subplan spawn** (#9131):
|
||||
Fixed `SubplanService.spawn()` to propagate all `invariant_enforced` decisions from the
|
||||
parent plan's decision tree to each child plan's decision tree. Previously, child plans
|
||||
started Strategize with a completely empty invariant set, violating the spec requirement:
|
||||
"recorded as `invariant_enforced` decisions that propagate to child plans." The fix adds
|
||||
a `_propagate_invariant_decisions()` helper that re-records each parent
|
||||
`invariant_enforced` decision on the child plan, including `non_overridable` global
|
||||
invariants. BDD regression coverage added in
|
||||
`features/tdd_invariant_propagation_subplan.feature`.
|
||||
|
||||
- **fix(repositories): derive PlanResult.success from result_success column instead of error_message** (#7501):
|
||||
Fixed a critical bug in `PlanRepository._to_domain` where `PlanResult.success` was incorrectly
|
||||
derived from `error_message is None`. Because `error_message` is shared between the build phase
|
||||
and the result phase, a plan with a historical build error would be marked as failed even after
|
||||
successfully completing and being applied. The fix introduces a dedicated `result_success` boolean
|
||||
column in the `plans` table (migration `m9_003_plan_result_success_column`) and updates the
|
||||
repository read path to use it. For backward compatibility, when `result_success` is NULL
|
||||
(pre-migration records), the legacy `error_message is None` heuristic is preserved.
|
||||
|
||||
- **`LLMTraceRepository.save()` premature commit breaks UnitOfWork transactions** (#7505):
|
||||
Replaced the unconditional `session.commit()` in `LLMTraceRepository.save()` with a
|
||||
dual-path implementation that respects the UnitOfWork (UoW) pattern. When an external
|
||||
@@ -286,6 +519,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
MCP logger thread-safety in `session.py` using a threading lock. Created
|
||||
migration guide for users transitioning from legacy to V3 workflow.
|
||||
|
||||
- **Plan Tree JSON/YAML Command Envelope** (#9163): `agents plan tree --format json/yaml`
|
||||
now wraps output in the spec-required command envelope with `command`, `status`,
|
||||
`exit_code`, `data`, `timing`, and `messages` fields. The `data` field contains
|
||||
`plan_id`, `tree`, `summary` (nodes, depth, child_plans, invariants, superseded),
|
||||
`child_plans` list, and `decision_ids` mapping. Timing now reflects actual elapsed
|
||||
milliseconds from command start to envelope construction.
|
||||
|
||||
- **Automation Profile Silent Fallback** (#8232): `_resolve_profile_for_plan` in
|
||||
`PlanLifecycleService` now raises a clear `ValidationError` when a plan's
|
||||
automation profile name is not a known built-in profile, instead of silently
|
||||
@@ -336,6 +576,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
forward-compatibility. Added BDD coverage for the stored-JSON path,
|
||||
corrupt-JSON fallback, resource-passing, and stub extra-kwargs scenarios. (#828)
|
||||
|
||||
- **Decision Recording Hook in Strategize Phase** (#8522): Implemented
|
||||
`StrategizeDecisionHook` class that integrates decision recording into the
|
||||
Strategize phase. The hook captures every decision point during strategy
|
||||
decomposition, including question, chosen option, alternatives considered,
|
||||
confidence score, rationale, and full context snapshot (hot context hash,
|
||||
actor state reference, relevant resources). Supports recording of
|
||||
`strategy_choice`, `resource_selection`, `subplan_spawn`, and
|
||||
`invariant_enforced` decision types. Context snapshots are auto-captured
|
||||
with SHA256 hashing of context data and checkpoint references for LangGraph
|
||||
actor state. Includes comprehensive BDD test suite with 40+ scenarios
|
||||
covering all decision types, context capture, error handling, and tree
|
||||
structure validation.
|
||||
|
||||
- **TDD Issue-Capture Test Activation** (#7025): Replaced 234 bare `@skip` tags
|
||||
across 82 Behave feature files with the correct `@tdd_expected_fail @tdd_issue
|
||||
@@ -507,6 +759,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
delays from 15s to 2s, and reduced idle sleep from 60s to 10s for dramatically
|
||||
faster throughput.
|
||||
|
||||
- **Specification — Validation Gate Empty-Run Guard** (#8146): Updated `docs/specification.md`
|
||||
to document the security invariant introduced in PR #7786 (fixing issue #7508). The spec now
|
||||
explicitly states that `ApplyValidationSummary.all_required_passed` returns `False` when no
|
||||
validations have been run (empty summary), blocking apply. Added a prominent danger admonition
|
||||
block, updated the validation process results section, the `final_validation_results` data
|
||||
model description, and two milestone acceptance criteria to reflect the corrected blocking
|
||||
behavior for empty validation summaries and no-attachment runs.
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -619,6 +878,15 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
response format from the OpenCode API `/session/status` endpoint instead of an array.
|
||||
Workers now dispatch and verify correctly, preventing incorrect session deletion.
|
||||
|
||||
---
|
||||
### Fixed
|
||||
|
||||
- **CLI (`agents actor remove`)** (#6491): Restores output parity with the
|
||||
other actor commands by honoring `--format`/`-f` for JSON/YAML/plain/Rich
|
||||
envelopes. Adds a Robot Framework regression test to assert the JSON
|
||||
envelope structure and updates the CLI synopsis in `docs/specification.md`
|
||||
to document the option.
|
||||
|
||||
---
|
||||
|
||||
## [3.8.0] -- 2026-04-05
|
||||
|
||||
+16
-3
@@ -5,6 +5,7 @@
|
||||
* HAL 9000 <hal9000@cleverthis.com>
|
||||
* Hamza Khyari <hamza.khyari@cleverthis.com>
|
||||
* Jeffrey Phillips Freeman <jeffrey.freeman@syncleus.com>
|
||||
* Jeffrey Phillips Freeman <the@jeffreyfreeman.me>
|
||||
* Luis Mendes <luis.p.mendes@gmail.com>
|
||||
* Rui Hu <rui.hu@cleverthis.com>
|
||||
|
||||
@@ -13,21 +14,33 @@
|
||||
Below are some of the specific details of various contributions.
|
||||
|
||||
* Jeffrey Phillips Freeman has acted as Lead Developer, daily contributor, and Project Owner.
|
||||
* Jeffrey Phillips Freeman has contributed an implementation for the invariant propagation fix (PR #10881 / issue #9131): added `_propagate_invariant_decisions()` to `SubplanService` to propagate all `invariant_enforced` decisions from parent plans to child plan decision trees during subplan spawn, satisfying the specification requirement for invariant propagation across hierarchical plan execution.
|
||||
* Brent E. Edwards has contributed quality assurance, test coverage, and CI pipeline improvements.
|
||||
* HAL 9000 has contributed automated implementation, bug fixes, and feature development as part of the CleverAgents automation pool.
|
||||
* HAL 9000 has contributed concurrency safety improvements, including thread-safe context tier management (issue #7547) for parallel plan execution.
|
||||
* HAL 9000 has contributed the plan concurrency race-condition fix (#7989): wired `LockService` into the plan lifecycle, guarding `execute_plan()` and `apply_plan()` with plan-level advisory locks and unique per-invocation owner identities to prevent silent concurrent state corruption.
|
||||
* HAL 9000 has contributed the bug-hunt-pool-supervisor non-blocking tracking fix: updated step 5 to be best-effort and added rule 9 to prevent the automation-tracking-manager call from blocking the main supervisor loop.
|
||||
<<<<<<< HEAD
|
||||
* HAL 9000 has contributed the bug-hunt-pool-supervisor non-blocking tracking fix (#7875 / PR #7957): updated step 5 to be best-effort and added rule 9 to prevent the automation-tracking-manager call from blocking the main supervisor loop.
|
||||
* Jeffrey Phillips Freeman has contributed the complete AUTO-BUG-POOL to AUTO-BUG-SUP tracking prefix fix across agent-system-specification.md, automation-tracking.md documentation and agent-system-specification.md spec document, replaced with correct `AUTO-BUG-SUP` prefix used by the bug-hunt-pool-supervisor agent (#7875).
|
||||
* HAL 9000 has contributed the plugin entry point security hardening fix (#7476): enforced entry point allowlist validation before importing plugin modules to prevent malicious plugin loading.
|
||||
* HAL 9000 has contributed the benchmark workflow separation (#9040): moved the benchmark-regression job out of the default PR workflow into a dedicated scheduled workflow, reducing median PR CI turnaround time from 99-132 minutes to under 30 minutes.
|
||||
* HAL 9000 has contributed the agent-evolution-pool-supervisor PR metadata assignment (#7888): the supervisor now automatically looks up the Type/Automation label and earliest open milestone before dispatching improvement PR creation workers, ensuring all generated improvement PRs have correct Type labels and milestone assignments.
|
||||
* HAL 9000 has contributed the plan tree JSON/YAML command envelope fix (#9163): wrapped `agents plan tree --format json/yaml` output in the spec-required command envelope structure, added summary statistics, decision_ids mapping, child_plans list, and accurate timing measurement.
|
||||
* HAL 9000 contributed Structural Component Output Validation (PR #11147): implemented `validate_plan_tree`, `validate_decision_dict`, `validate_structured_output`, and `validate_structured_component_output` validators that replace exact-character matching with structural schema checking for plan tree nodes, decision CLI dictionaries, and structured session output envelopes.
|
||||
* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc.
|
||||
* HAL 9000 has contributed automated bug fixes, CLI output formatting improvements, and ongoing maintenance as part of the CleverAgents automation system.
|
||||
* HAL 9000 has contributed the file edit encoding parameter fix (PR #8258 / issue #7559).
|
||||
<<* HAL 9000 has contributed the architecture-pool-supervisor milestone assignment feature (PR #8188 / issue #7521): added `forgejo_update_pull_request` permission and documented the PR workflow for major spec changes, enabling automatic milestone assignment for specification PRs.
|
||||
* HAL 9000 has contributed the architecture-pool-supervisor milestone assignment feature (PR #8188 / issue #7521): added `forgejo_update_pull_request` permission and documented the PR workflow for major spec changes, enabling automatic milestone assignment for specification PRs.
|
||||
* HAL 9000 has contributed the git worktree TOCTOU race condition fix (PR #8178 / issue #7507): replaced the unsafe mkdtemp() + rmdir() pattern with a parent-directory approach to eliminate the race window in concurrent git worktree operations.
|
||||
* HAL 9000 has contributed the git_tools TOCTOU race condition fix (PR #8255 / issue #7619): eliminated the Time-Of-Check-To-Time-Of-Use race in `_get_base_env()` by adding double-checked locking with a module-level `threading.Lock`, preventing concurrent threads from writing conflicting environment snapshots.
|
||||
* HAL 9000 has contributed the mandatory PR compliance checklist to `implementation-supervisor.md` (#9824): added an 8-item checklist to the worker prompt body with concrete items covering CHANGELOG.md, CONTRIBUTORS.md, commit footer, CI verification, BDD tests, Epic reference, labels, and milestone assignment to eliminate systemic PR merge blockers.
|
||||
* HAL 9000 has contributed the PlanResult.success derivation fix (PR #8214 / issue #7501): replaced the incorrect `error_message is None` heuristic with a dedicated `result_success` column in the plans table, ensuring plans with historical build errors are not incorrectly marked as failed after a successful apply.
|
||||
* HAL 9000 has contributed the mandatory PR compliance checklist to `implementation-pool-supervisor.md` (#9824): created a new agent definition with an embedded 8-item checklist ensuring workers always update CHANGELOG.md, CONTRIBUTORS.md, include commit footers (`ISSUES CLOSED: #N`), verify CI passes, add BDD tests, reference the parent Epic, apply labels via forgejo-label-manager, and assign milestones before creating PRs. Includes concrete examples for each subsection and compliance verification pseudocode.
|
||||
* HAL 9000 has contributed comprehensive milestone documentation for v3.6.0 (Advanced Concepts & Deferred Features) and v3.7.0 (TUI Implementation) (PR #9903): split into sub-documents covering context strategies, LLM backends, resource types, A2A rename, container tool execution, scope chain resolution, cost/safety budgets, E2E workflow tests, code review examples, plugin architecture, TUI layout, persona system, reference/command input, session management, configuration, and TuiMaterializer integration.
|
||||
* HAL 9000 has contributed the LLMTraceRepository data-integrity fix (PR #8185 / issue #7505): replaced the unconditional `session.commit()` in `LLMTraceRepository.save()` with a dual-path implementation that respects the UnitOfWork pattern — flushing only when an external session is provided, and flushing + committing + closing when operating standalone. This eliminates premature transaction commits, loss of rollback capability, and a docstring/implementation mismatch.
|
||||
* HAL 9000 has contributed the ACMS Index Data Model and File Traversal Engine (PR #9664 / issue #9579): foundational data structures for indexed context entries with hot/warm/cold/archive storage tier classification, tag system, and a timeout-safe chunked file traversal engine for large projects with 10,000+ files.
|
||||
* HAL 9000 has contributed the error-suppression removal fix (PR #9247 / issue #9060): removed both `try...except Exception:` blocks in `register_registry_agents()` that silently suppressed errors from `actor_registry.list_actors()` and the route bridge refresh, enabling exceptions to propagate per CONTRIBUTING.md fail-fast policy. Added three Behave scenarios verifying RuntimeError, AttributeError, and TypeError propagation.
|
||||
* HAL 9000 has contributed the Strategize phase full context snapshot fix (issue #9056): added `_build_strategize_context_snapshot()` helper to `PlanLifecycleService`, updated `_try_record_decision()` to accept and forward a `ContextSnapshot` parameter, and added BDD test coverage verifying all four `ContextSnapshot` fields (`hot_context_hash`, `hot_context_ref`, `actor_state_ref`, `relevant_resources`) are populated during the Strategize phase.
|
||||
* HAL 9000 has contributed the ACMS context path matching fix (PR #10975 / issue #10972): corrects `_path_matches()` and `_matches_pattern()` to properly match absolute fragment paths against relative glob patterns by auto-prefixing with `**/` before calling `PurePath.full_match()`, preventing silent inefficacy of include/exclude filters for absolute paths in fragment metadata.
|
||||
* HAL 9000 has contributed database resource types (PostgreSQL, SQLite) with transaction-based sandbox strategy: implemented ``DatabaseResourceHandler`` providing full CRUD operations (`read`, `write`, `delete`, `list_children`) and connection validation with automatic credential masking for PostgreSQL and SQLite backends. Includes ``TransactionSandbox`` infrastructure wired into ``SandboxFactory``, BDD test coverage in ``features/database_resources.feature``, and Robot Framework integration tests in ``robot/database_resources.robot`` (PR #10591 / issue #8608, Epic #8568).
|
||||
* HAL 9000 has contributed the agents plan rollback command (PR #8674 / issue #8557): implemented checkpoint-based plan state restoration with the `agents plan rollback <plan-id> [<checkpoint-id>]` CLI command as part of Epic #8493, enabling plans to be restored to previous checkpoints, discarding post-checkpoint decisions, and resuming execution from the rolled-back state. Supported by `--yes/-y`, `--to-checkpoint`, and `--format/-f` flags. Includes comprehensive BDD test coverage (>= 97%) for rollback, decision discarding, and plan resume functionality.
|
||||
* HAL 9000 has contributed the PyYAML security upgrade (PR #11012 / issue #9055): added `pyyaml>=6.0.3` dependency constraint to address known YAML parsing vulnerabilities.
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
"""ASV benchmarks for circuit breaker pattern.
|
||||
|
||||
Measures the overhead of circuit breaker state management,
|
||||
state transitions, and fast-fail latency in open state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from cleveragents.core.circuit_breaker import (
|
||||
CircuitBreaker,
|
||||
CircuitBreakerOpen,
|
||||
CircuitBreakerState,
|
||||
)
|
||||
|
||||
|
||||
class CircuitBreakerClosedStateBench:
|
||||
"""Benchmark circuit breaker overhead in closed state."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up circuit breaker in closed state."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=5,
|
||||
recovery_timeout=60.0,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=2,
|
||||
cooldown_seconds=30.0,
|
||||
name="test_service",
|
||||
)
|
||||
|
||||
def time_call_success(self) -> None:
|
||||
"""Benchmark successful call overhead in closed state."""
|
||||
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
self.breaker.call(dummy_func)
|
||||
|
||||
def time_call_with_args(self) -> None:
|
||||
"""Benchmark call with arguments in closed state."""
|
||||
|
||||
def dummy_func(x: int, y: str) -> str:
|
||||
return f"{x}:{y}"
|
||||
|
||||
self.breaker.call(dummy_func, 42, "test")
|
||||
|
||||
def time_state_check(self) -> None:
|
||||
"""Benchmark state property access."""
|
||||
_ = self.breaker.state
|
||||
|
||||
def time_failure_count_check(self) -> None:
|
||||
"""Benchmark failure count property access."""
|
||||
_ = self.breaker.failure_count
|
||||
|
||||
|
||||
class CircuitBreakerOpenStateBench:
|
||||
"""Benchmark circuit breaker fast-fail latency in open state."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up circuit breaker in open state."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=2,
|
||||
recovery_timeout=60.0,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=2,
|
||||
cooldown_seconds=30.0,
|
||||
name="test_service",
|
||||
)
|
||||
# Force the breaker into open state
|
||||
for _ in range(3):
|
||||
try:
|
||||
self.breaker.call(lambda: 1 / 0) # Raise ZeroDivisionError
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
|
||||
def time_fast_fail(self) -> None:
|
||||
"""Benchmark fast-fail latency when circuit is open."""
|
||||
try:
|
||||
self.breaker.call(lambda: "should not execute")
|
||||
except CircuitBreakerOpen:
|
||||
pass
|
||||
|
||||
def time_open_state_check(self) -> None:
|
||||
"""Benchmark state check when open."""
|
||||
assert self.breaker.state == CircuitBreakerState.OPEN
|
||||
|
||||
|
||||
class CircuitBreakerClosedToOpenTransitionBench:
|
||||
"""Benchmark closed-to-open state transition overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up a fresh circuit breaker in closed state."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=2,
|
||||
recovery_timeout=60.0,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=1,
|
||||
cooldown_seconds=30.0,
|
||||
name="test_service",
|
||||
)
|
||||
|
||||
def time_closed_to_open_transition(self) -> None:
|
||||
"""Benchmark transition from closed to open state."""
|
||||
# Trigger failures to transition to open
|
||||
for _ in range(3):
|
||||
try:
|
||||
self.breaker.call(lambda: 1 / 0)
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
|
||||
|
||||
class CircuitBreakerOpenToHalfOpenTransitionBench:
|
||||
"""Benchmark open-to-half-open state transition overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up circuit breaker already in open state."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=2,
|
||||
recovery_timeout=0.05, # Very short timeout
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=1,
|
||||
cooldown_seconds=0.01,
|
||||
name="test_service",
|
||||
)
|
||||
# Force to open state
|
||||
for _ in range(3):
|
||||
try:
|
||||
self.breaker.call(lambda: 1 / 0)
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
# Wait for recovery timeout so transition to half-open is possible
|
||||
time.sleep(0.1)
|
||||
|
||||
def time_open_to_half_open_transition(self) -> None:
|
||||
"""Benchmark transition from open to half-open state."""
|
||||
# Attempt call to trigger half-open transition
|
||||
try:
|
||||
self.breaker.call(lambda: "success")
|
||||
except CircuitBreakerOpen:
|
||||
pass
|
||||
|
||||
|
||||
class CircuitBreakerHalfOpenToClosedTransitionBench:
|
||||
"""Benchmark half-open-to-closed state transition overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up circuit breaker already in half-open state."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=2,
|
||||
recovery_timeout=0.05,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=1,
|
||||
cooldown_seconds=0.01,
|
||||
name="test_service",
|
||||
)
|
||||
# Force to open state
|
||||
for _ in range(3):
|
||||
try:
|
||||
self.breaker.call(lambda: 1 / 0)
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
# Wait for recovery timeout so breaker enters half-open on next call
|
||||
time.sleep(0.1)
|
||||
|
||||
def time_half_open_to_closed_transition(self) -> None:
|
||||
"""Benchmark transition from half-open to closed state."""
|
||||
# Successful call in half-open should close the breaker
|
||||
self.breaker.call(lambda: "success")
|
||||
|
||||
|
||||
class CircuitBreakerAsyncBench:
|
||||
"""Benchmark async circuit breaker operations."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up async circuit breakers."""
|
||||
self.breaker = CircuitBreaker(
|
||||
failure_threshold=5,
|
||||
recovery_timeout=60.0,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=2,
|
||||
cooldown_seconds=30.0,
|
||||
name="test_service",
|
||||
)
|
||||
# Set up a separate breaker already in open state for fast-fail benchmark
|
||||
self.open_breaker = CircuitBreaker(
|
||||
failure_threshold=2,
|
||||
recovery_timeout=60.0,
|
||||
expected_exception=Exception,
|
||||
half_open_max_successes=2,
|
||||
cooldown_seconds=30.0,
|
||||
name="test_service_open",
|
||||
)
|
||||
for _ in range(3):
|
||||
try:
|
||||
self.open_breaker.call(lambda: 1 / 0)
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
|
||||
def time_async_call_success(self) -> None:
|
||||
"""Benchmark successful async call overhead."""
|
||||
|
||||
async def dummy_async() -> str:
|
||||
return "success"
|
||||
|
||||
asyncio.run(self.breaker.async_call(dummy_async))
|
||||
|
||||
def time_async_call_with_args(self) -> None:
|
||||
"""Benchmark async call with arguments."""
|
||||
|
||||
async def dummy_async(x: int, y: str) -> str:
|
||||
return f"{x}:{y}"
|
||||
|
||||
asyncio.run(self.breaker.async_call(dummy_async, 42, "test"))
|
||||
|
||||
def time_async_fast_fail(self) -> None:
|
||||
"""Benchmark async fast-fail when open."""
|
||||
|
||||
async def dummy_async() -> str:
|
||||
return "should not execute"
|
||||
|
||||
try:
|
||||
asyncio.run(self.open_breaker.async_call(dummy_async))
|
||||
except CircuitBreakerOpen:
|
||||
pass
|
||||
|
||||
|
||||
class CircuitBreakerInitializationBench:
|
||||
"""Benchmark circuit breaker initialization overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_default_initialization(self) -> None:
|
||||
"""Benchmark default initialization."""
|
||||
CircuitBreaker()
|
||||
|
||||
def time_custom_initialization(self) -> None:
|
||||
"""Benchmark initialization with custom parameters."""
|
||||
CircuitBreaker(
|
||||
failure_threshold=10,
|
||||
recovery_timeout=120.0,
|
||||
expected_exception=ValueError,
|
||||
half_open_max_successes=3,
|
||||
cooldown_seconds=45.0,
|
||||
name="custom_service",
|
||||
)
|
||||
|
||||
def time_multiple_breakers(self) -> None:
|
||||
"""Benchmark creating multiple breakers."""
|
||||
for i in range(10):
|
||||
CircuitBreaker(name=f"service_{i}")
|
||||
@@ -0,0 +1,280 @@
|
||||
"""ASV benchmarks for retry patterns.
|
||||
|
||||
Measures the overhead of retry decorator construction and
|
||||
happy-path invocation for the public retry factory functions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from cleveragents.core.retry_patterns import (
|
||||
get_retry_decorator,
|
||||
retry_on_result,
|
||||
retry_with_exponential_backoff,
|
||||
retry_with_jitter,
|
||||
retry_with_timeout,
|
||||
should_retry_result,
|
||||
)
|
||||
|
||||
|
||||
class RetryDecoratorConstructionBench:
|
||||
"""Benchmark retry decorator construction overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_retry_with_exponential_backoff_construction(self) -> None:
|
||||
"""Benchmark constructing exponential backoff decorator."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=3)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_retry_with_jitter_construction(self) -> None:
|
||||
"""Benchmark constructing jitter decorator."""
|
||||
|
||||
@retry_with_jitter(max_attempts=3)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_retry_with_timeout_construction(self) -> None:
|
||||
"""Benchmark constructing timeout decorator."""
|
||||
|
||||
@retry_with_timeout(timeout_seconds=5.0)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_retry_on_result_construction(self) -> None:
|
||||
"""Benchmark constructing result-based retry decorator."""
|
||||
|
||||
@retry_on_result(should_retry_result, max_attempts=3)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_get_retry_decorator_network(self) -> None:
|
||||
"""Benchmark constructing decorator via factory for network category."""
|
||||
get_retry_decorator("network")
|
||||
|
||||
def time_get_retry_decorator_provider(self) -> None:
|
||||
"""Benchmark constructing decorator via factory for provider category."""
|
||||
get_retry_decorator("provider")
|
||||
|
||||
def time_get_retry_decorator_database(self) -> None:
|
||||
"""Benchmark constructing decorator via factory for database category."""
|
||||
get_retry_decorator("database")
|
||||
|
||||
def time_get_retry_decorator_unknown(self) -> None:
|
||||
"""Benchmark constructing decorator via factory for unknown category."""
|
||||
get_retry_decorator("unknown_category")
|
||||
|
||||
|
||||
class RetryDecoratorInvocationBench:
|
||||
"""Benchmark happy-path retry decorator invocation."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up decorated functions."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=3)
|
||||
def exponential_func() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_with_jitter(max_attempts=3)
|
||||
def jitter_func() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_with_timeout(timeout_seconds=5.0)
|
||||
def timeout_func() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_on_result(should_retry_result, max_attempts=3)
|
||||
def result_func() -> str:
|
||||
return "success"
|
||||
|
||||
self.exponential_func = exponential_func
|
||||
self.jitter_func = jitter_func
|
||||
self.timeout_func = timeout_func
|
||||
self.result_func = result_func
|
||||
|
||||
def time_exponential_backoff_invocation(self) -> None:
|
||||
"""Benchmark exponential backoff invocation (happy path)."""
|
||||
self.exponential_func()
|
||||
|
||||
def time_jitter_invocation(self) -> None:
|
||||
"""Benchmark jitter invocation (happy path)."""
|
||||
self.jitter_func()
|
||||
|
||||
def time_timeout_invocation(self) -> None:
|
||||
"""Benchmark timeout invocation (happy path)."""
|
||||
self.timeout_func()
|
||||
|
||||
def time_result_based_invocation(self) -> None:
|
||||
"""Benchmark result-based retry invocation (happy path)."""
|
||||
self.result_func()
|
||||
|
||||
|
||||
class RetryDecoratorAsyncBench:
|
||||
"""Benchmark async retry decorator operations."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up async decorated functions."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=3)
|
||||
async def async_exponential() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_with_jitter(max_attempts=3)
|
||||
async def async_jitter() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_with_timeout(timeout_seconds=5.0)
|
||||
async def async_timeout() -> str:
|
||||
return "success"
|
||||
|
||||
self.async_exponential = async_exponential
|
||||
self.async_jitter = async_jitter
|
||||
self.async_timeout = async_timeout
|
||||
|
||||
def time_async_exponential_invocation(self) -> None:
|
||||
"""Benchmark async exponential backoff invocation."""
|
||||
asyncio.run(self.async_exponential())
|
||||
|
||||
def time_async_jitter_invocation(self) -> None:
|
||||
"""Benchmark async jitter invocation."""
|
||||
asyncio.run(self.async_jitter())
|
||||
|
||||
def time_async_timeout_invocation(self) -> None:
|
||||
"""Benchmark async timeout invocation."""
|
||||
asyncio.run(self.async_timeout())
|
||||
|
||||
|
||||
class RetryDecoratorWithArgumentsBench:
|
||||
"""Benchmark retry decorators with function arguments."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up decorated functions with arguments."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=3)
|
||||
def func_with_args(x: int, y: str, z: float = 1.0) -> str:
|
||||
return f"{x}:{y}:{z}"
|
||||
|
||||
@retry_with_jitter(max_attempts=3)
|
||||
def func_with_kwargs(a: int, b: str = "default") -> str:
|
||||
return f"{a}:{b}"
|
||||
|
||||
self.func_with_args = func_with_args
|
||||
self.func_with_kwargs = func_with_kwargs
|
||||
|
||||
def time_exponential_with_positional_args(self) -> None:
|
||||
"""Benchmark exponential backoff with positional arguments."""
|
||||
self.func_with_args(42, "test", 2.5)
|
||||
|
||||
def time_exponential_with_keyword_args(self) -> None:
|
||||
"""Benchmark exponential backoff with keyword arguments."""
|
||||
self.func_with_args(x=42, y="test", z=2.5)
|
||||
|
||||
def time_jitter_with_mixed_args(self) -> None:
|
||||
"""Benchmark jitter with mixed positional and keyword arguments."""
|
||||
self.func_with_kwargs(100, b="custom")
|
||||
|
||||
|
||||
class RetryDecoratorFactoryBench:
|
||||
"""Benchmark retry decorator factory function."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_factory_network_category(self) -> None:
|
||||
"""Benchmark factory with network category."""
|
||||
decorator = get_retry_decorator("network")
|
||||
|
||||
@decorator
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_factory_provider_category(self) -> None:
|
||||
"""Benchmark factory with provider category."""
|
||||
decorator = get_retry_decorator("provider")
|
||||
|
||||
@decorator
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_factory_database_category(self) -> None:
|
||||
"""Benchmark factory with database category."""
|
||||
decorator = get_retry_decorator("database")
|
||||
|
||||
@decorator
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_factory_file_operation_category(self) -> None:
|
||||
"""Benchmark factory with file_operation category."""
|
||||
decorator = get_retry_decorator("file_operation")
|
||||
|
||||
@decorator
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
|
||||
class RetryDecoratorConfigurationBench:
|
||||
"""Benchmark retry decorator with various configurations."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_minimal_config(self) -> None:
|
||||
"""Benchmark decorator with minimal configuration."""
|
||||
|
||||
@retry_with_exponential_backoff()
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_aggressive_config(self) -> None:
|
||||
"""Benchmark decorator with aggressive retry settings."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=10)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_conservative_config(self) -> None:
|
||||
"""Benchmark decorator with conservative retry settings."""
|
||||
|
||||
@retry_with_exponential_backoff(max_attempts=1)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_long_timeout_config(self) -> None:
|
||||
"""Benchmark decorator with long timeout."""
|
||||
|
||||
@retry_with_timeout(timeout_seconds=300.0)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_short_timeout_config(self) -> None:
|
||||
"""Benchmark decorator with short timeout."""
|
||||
|
||||
@retry_with_timeout(timeout_seconds=0.1)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
@@ -0,0 +1,367 @@
|
||||
"""ASV benchmarks for service-level retry patterns.
|
||||
|
||||
Measures the overhead of retry_service_operation decorator
|
||||
and retry_auto_debug happy-path latency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from cleveragents.core.circuit_breaker import CircuitBreaker
|
||||
from cleveragents.core.retry_service_patterns import (
|
||||
RetryContext,
|
||||
is_read_only_plan_operation,
|
||||
retry_auto_debug,
|
||||
retry_service_operation,
|
||||
)
|
||||
|
||||
|
||||
class RetryServiceOperationDecoratorBench:
|
||||
"""Benchmark retry_service_operation decorator overhead."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_decorator_construction_sync(self) -> None:
|
||||
"""Benchmark constructing sync service retry decorator."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_decorator_construction_async(self) -> None:
|
||||
"""Benchmark constructing async service retry decorator."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
async def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_decorator_with_circuit_breaker(self) -> None:
|
||||
"""Benchmark decorator with circuit breaker instance."""
|
||||
cb = CircuitBreaker(
|
||||
failure_threshold=5,
|
||||
recovery_timeout=60.0,
|
||||
name="test_service",
|
||||
)
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
circuit_breaker=cb,
|
||||
)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_decorator_with_exponential_strategy(self) -> None:
|
||||
"""Benchmark decorator with exponential backoff strategy."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
backoff_strategy="exponential",
|
||||
base_delay=1.0,
|
||||
max_delay=60.0,
|
||||
)
|
||||
def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
|
||||
class RetryServiceOperationInvocationBench:
|
||||
"""Benchmark happy-path service retry invocation."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up decorated service operations."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
def sync_op() -> str:
|
||||
return "success"
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
async def async_op() -> str:
|
||||
return "success"
|
||||
|
||||
cb = CircuitBreaker(
|
||||
failure_threshold=5,
|
||||
recovery_timeout=60.0,
|
||||
name="test_service",
|
||||
)
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
circuit_breaker=cb,
|
||||
)
|
||||
def breaker_op() -> str:
|
||||
return "success"
|
||||
|
||||
self.sync_op = sync_op
|
||||
self.async_op = async_op
|
||||
self.breaker_op = breaker_op
|
||||
|
||||
def time_sync_operation_invocation(self) -> None:
|
||||
"""Benchmark sync service operation invocation (happy path)."""
|
||||
self.sync_op()
|
||||
|
||||
def time_async_operation_invocation(self) -> None:
|
||||
"""Benchmark async service operation invocation (happy path)."""
|
||||
asyncio.run(self.async_op())
|
||||
|
||||
def time_operation_with_circuit_breaker(self) -> None:
|
||||
"""Benchmark service operation with circuit breaker (happy path)."""
|
||||
self.breaker_op()
|
||||
|
||||
|
||||
class RetryServiceOperationWithArgumentsBench:
|
||||
"""Benchmark service retry with function arguments."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Set up decorated operations with arguments."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="fetch_data",
|
||||
max_attempts=3,
|
||||
)
|
||||
def fetch_data(
|
||||
resource_id: int, include_metadata: bool = False
|
||||
) -> dict[str, Any]:
|
||||
return {"id": resource_id, "data": "test"}
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="update_data",
|
||||
max_attempts=3,
|
||||
)
|
||||
async def update_data(
|
||||
resource_id: int, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return {"id": resource_id, "updated": True}
|
||||
|
||||
self.fetch_data = fetch_data
|
||||
self.update_data = update_data
|
||||
|
||||
def time_sync_with_positional_args(self) -> None:
|
||||
"""Benchmark sync operation with positional arguments."""
|
||||
self.fetch_data(42, True)
|
||||
|
||||
def time_sync_with_keyword_args(self) -> None:
|
||||
"""Benchmark sync operation with keyword arguments."""
|
||||
self.fetch_data(resource_id=42, include_metadata=True)
|
||||
|
||||
def time_async_with_complex_payload(self) -> None:
|
||||
"""Benchmark async operation with complex payload."""
|
||||
payload = {"key": "value", "nested": {"data": [1, 2, 3]}}
|
||||
asyncio.run(self.update_data(42, payload))
|
||||
|
||||
|
||||
class RetryAutoDebugBench:
|
||||
"""Benchmark retry_auto_debug decorator."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_auto_debug_construction(self) -> None:
|
||||
"""Benchmark constructing auto-debug decorator."""
|
||||
|
||||
@retry_auto_debug(max_debug_attempts=3)
|
||||
async def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
def time_auto_debug_invocation(self) -> None:
|
||||
"""Benchmark auto-debug invocation (happy path)."""
|
||||
|
||||
@retry_auto_debug(max_debug_attempts=3)
|
||||
async def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
asyncio.run(dummy_func())
|
||||
|
||||
def time_auto_debug_single_attempt(self) -> None:
|
||||
"""Benchmark auto-debug with single attempt."""
|
||||
|
||||
@retry_auto_debug(max_debug_attempts=1)
|
||||
async def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
asyncio.run(dummy_func())
|
||||
|
||||
def time_auto_debug_multiple_attempts(self) -> None:
|
||||
"""Benchmark auto-debug with multiple attempts configured."""
|
||||
|
||||
@retry_auto_debug(max_debug_attempts=5)
|
||||
async def dummy_func() -> str:
|
||||
return "success"
|
||||
|
||||
asyncio.run(dummy_func())
|
||||
|
||||
|
||||
class RetryContextBench:
|
||||
"""Benchmark RetryContext operations."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_context_creation(self) -> None:
|
||||
"""Benchmark creating a retry context."""
|
||||
RetryContext(
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
|
||||
def time_context_with_custom_wait(self) -> None:
|
||||
"""Benchmark context with custom wait strategy."""
|
||||
from tenacity import wait_exponential
|
||||
|
||||
RetryContext(
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
wait_strategy=wait_exponential(multiplier=1, min=1, max=30),
|
||||
)
|
||||
|
||||
def time_context_property_access(self) -> None:
|
||||
"""Benchmark accessing context properties."""
|
||||
ctx = RetryContext(
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
_ = ctx.operation_name
|
||||
_ = ctx.max_attempts
|
||||
_ = ctx.attempt_count
|
||||
|
||||
def time_context_execute(self) -> None:
|
||||
"""Benchmark executing a function via RetryContext."""
|
||||
ctx = RetryContext(
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
)
|
||||
ctx.execute(lambda: "success")
|
||||
|
||||
|
||||
class IsReadOnlyPlanOperationBench:
|
||||
"""Benchmark is_read_only_plan_operation utility."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_check_strategize(self) -> None:
|
||||
"""Benchmark checking strategize operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "strategize"})
|
||||
|
||||
def time_check_plan(self) -> None:
|
||||
"""Benchmark checking plan operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "plan"})
|
||||
|
||||
def time_check_validate(self) -> None:
|
||||
"""Benchmark checking validate operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "validate"})
|
||||
|
||||
def time_check_review(self) -> None:
|
||||
"""Benchmark checking review operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "review"})
|
||||
|
||||
def time_check_preview(self) -> None:
|
||||
"""Benchmark checking preview operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "preview"})
|
||||
|
||||
def time_check_check(self) -> None:
|
||||
"""Benchmark checking check operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "check"})
|
||||
|
||||
def time_check_execute(self) -> None:
|
||||
"""Benchmark checking execute operation (not read-only)."""
|
||||
is_read_only_plan_operation({"plan_phase": "execute"})
|
||||
|
||||
def time_check_read_only_flag(self) -> None:
|
||||
"""Benchmark checking read_only flag."""
|
||||
is_read_only_plan_operation({"read_only": True})
|
||||
|
||||
def time_check_unknown_operation(self) -> None:
|
||||
"""Benchmark checking unknown operation."""
|
||||
is_read_only_plan_operation({"plan_phase": "unknown_op"})
|
||||
|
||||
def time_check_empty_kwargs(self) -> None:
|
||||
"""Benchmark checking empty kwargs."""
|
||||
is_read_only_plan_operation({})
|
||||
|
||||
|
||||
class RetryServiceOperationStrategyBench:
|
||||
"""Benchmark different retry strategies in service operations."""
|
||||
|
||||
timeout = 60
|
||||
|
||||
def time_exponential_strategy(self) -> None:
|
||||
"""Benchmark exponential strategy."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
backoff_strategy="exponential",
|
||||
)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_linear_strategy(self) -> None:
|
||||
"""Benchmark linear strategy."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
backoff_strategy="linear",
|
||||
)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_fixed_strategy(self) -> None:
|
||||
"""Benchmark fixed strategy."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
backoff_strategy="fixed",
|
||||
)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
|
||||
def time_jitter_strategy(self) -> None:
|
||||
"""Benchmark jitter strategy."""
|
||||
|
||||
@retry_service_operation(
|
||||
service_name="test_service",
|
||||
operation_name="test_op",
|
||||
max_attempts=3,
|
||||
backoff_strategy="jitter",
|
||||
)
|
||||
def dummy() -> str:
|
||||
return "success"
|
||||
|
||||
dummy()
|
||||
@@ -665,7 +665,7 @@ The Automation Tracking Manager provides eleven operations:
|
||||
| `AUTO-IMP-POOL` | Implementation Pool Supervisor |
|
||||
| `AUTO-REV-POOL` | PR Review Pool Supervisor |
|
||||
| `AUTO-UAT-POOL` | UAT Test Pool Supervisor |
|
||||
| `AUTO-BUG-POOL` | Bug Hunt Pool Supervisor |
|
||||
| `AUTO-BUG-SUP` | Bug Hunt Pool Supervisor |
|
||||
| `AUTO-INF-POOL` | Test Infrastructure Pool Supervisor |
|
||||
| `AUTO-ARCH` | Architecture Supervisor |
|
||||
| `AUTO-EPIC` | Epic Planning Supervisor |
|
||||
@@ -693,7 +693,7 @@ The following table maps between the two schemes for agents where they differ:
|
||||
| Implementation Pool | `AUTO-IMP-SUP` | `AUTO-IMP-POOL` |
|
||||
| PR Review Pool | `AUTO-REV-SUP` | `AUTO-REV-POOL` |
|
||||
| UAT Test Pool | `AUTO-UAT-SUP` | `AUTO-UAT-POOL` |
|
||||
| Bug Hunt Pool | `AUTO-BUG-SUP` | `AUTO-BUG-POOL` |
|
||||
| Bug Hunt Pool | `AUTO-BUG-SUP` | `AUTO-BUG-SUP` |
|
||||
| Test Infrastructure Pool | `AUTO-INF-SUP` | `AUTO-INF-POOL` |
|
||||
| Human Liaison | `AUTO-HUMAN` | `AUTO-LIAISON` |
|
||||
| Grooming | `AUTO-GROOM` | `AUTO-GROOMER` |
|
||||
@@ -1869,7 +1869,7 @@ Only critical bugs get assigned to the active milestone. Non-critical issues go
|
||||
| **Mode** | `subagent` |
|
||||
| **Model** | `google/gemini-2.5-pro` |
|
||||
| **Temperature** | 0.1 |
|
||||
| **Tracking Prefix** | `AUTO-BUG-POOL` |
|
||||
| **Tracking Prefix** | `AUTO-BUG-SUP` |
|
||||
| **Worker Count** | N/4 (quarter allocation) |
|
||||
|
||||
#### 8.6.1 Purpose
|
||||
@@ -3164,7 +3164,7 @@ Defines how ALL agents discover, interact with, and coordinate through the autom
|
||||
|
||||
### 13.9 Bug Hunter Tracking Update (`shared/bug_hunter_tracking_update.md`)
|
||||
|
||||
Defines the migration guide for converting the Bug Hunt supervisor from the old session state system to the new individual tracking issue system. Documents the tracking issue format `[AUTO-BUG-POOL] Bug Detection Pool Status (Cycle N)`, the cleanup protocol (one issue per cycle, preserve announcements), and worker coordination via announcement issues.
|
||||
Defines the migration guide for converting the Bug Hunt supervisor from the old session state system to the new individual tracking issue system. Documents the tracking issue format `[AUTO-BUG-SUP] Bug Hunt Status (Cycle 25)`, the cleanup protocol (one issue per cycle, preserve announcements), and worker coordination via announcement issues.
|
||||
|
||||
---
|
||||
|
||||
@@ -6767,7 +6767,7 @@ The system uses five distinct redundancy patterns:
|
||||
| 4 | pr-merge-pool-supervisor | subagent | claude-sonnet-4-6 | AUTO-MERGE | Singleton Supervisor |
|
||||
| 5 | pr-fix-pool-supervisor | *(removed — absorbed into implementation-pool-supervisor)* | -- | -- | -- |
|
||||
| 6 | uat-test-pool-supervisor | subagent | claude-sonnet-4-6 | AUTO-UAT-POOL | Pool Supervisor |
|
||||
| 7 | bug-hunt-pool-supervisor | subagent | gemini-2.5-pro | AUTO-BUG-POOL | Pool Supervisor |
|
||||
| 7 | bug-hunt-pool-supervisor | subagent | gemini-2.5-pro | AUTO-BUG-SUP | Pool Supervisor |
|
||||
| 8 | test-infra-pool-supervisor | subagent | gemini-2.5-pro | AUTO-INF-POOL | Pool Supervisor |
|
||||
| 9 | architecture-pool-supervisor | subagent | claude-sonnet-4-6 | AUTO-ARCH | Singleton Supervisor |
|
||||
| 10 | epic-planning-pool-supervisor | subagent | claude-sonnet-4-6 | AUTO-EPIC | Singleton Supervisor |
|
||||
|
||||
@@ -61,7 +61,7 @@ All prefixes are registered in the `automation-tracking-manager` subagent, which
|
||||
| uat-test-pool-supervisor | `AUTO-UAT-POOL` | `[AUTO-UAT-POOL] UAT Status (Cycle 6)` |
|
||||
| project-owner-pool-supervisor | `AUTO-PROJ-OWN` | `[AUTO-PROJ-OWN] Project Status (Cycle 11)` |
|
||||
| agent-evolution-pool-supervisor | `AUTO-EVLV` | `[AUTO-EVLV] Agent Evolution Report (Cycle 10)` |
|
||||
| bug-hunt-pool-supervisor | `AUTO-BUG-POOL` | `[AUTO-BUG-POOL] Bug Detection Pool Status (Cycle 60)` |
|
||||
| bug-hunt-pool-supervisor | `AUTO-BUG-SUP` | `[AUTO-BUG-SUP] Bug Hunt Status (Cycle 25)` |
|
||||
| spec-update-pool-supervisor | `AUTO-SPEC` | `[AUTO-SPEC] Specification Update Report (Cycle 1)` |
|
||||
| test-infra-pool-supervisor | `AUTO-INF-POOL` | `[AUTO-INF-POOL] Infrastructure Analysis Report (Cycle 2)` |
|
||||
| epic-planner | `AUTO-EPIC` | `[AUTO-EPIC] Epic Planning Update (Cycle 9)` |
|
||||
@@ -394,7 +394,7 @@ label:"Automation Tracking" [AUTO-UAT-POOL] in:title
|
||||
label:"Automation Tracking" [AUTO-PROJ-OWN] in:title
|
||||
label:"Automation Tracking" [AUTO-EVLV] in:title
|
||||
label:"Automation Tracking" [AUTO-EPIC] in:title
|
||||
label:"Automation Tracking" [AUTO-BUG-POOL] in:title
|
||||
label:"Automation Tracking" [AUTO-BUG-SUP] in:title
|
||||
label:"Automation Tracking" [AUTO-SPEC] in:title
|
||||
label:"Automation Tracking" [AUTO-INF-POOL] in:title
|
||||
```
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Quick Start Guide
|
||||
|
||||
This quick start guide will walk you through creating a new project, registering a resource, running a plan, and applying changes with CleverAgents.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Python 3.11+ and virtualenv
|
||||
- Git
|
||||
- A working CleverAgents installation (see development/testing.md for details)
|
||||
|
||||
## Install (local development)
|
||||
|
||||
```bash
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -e .[dev]
|
||||
```
|
||||
|
||||
## Create a new project
|
||||
|
||||
```bash
|
||||
# Create a new directory for your project
|
||||
mkdir my-project && cd my-project
|
||||
# Initialize a CleverAgents project (example command)
|
||||
cleveragents init --name my-project
|
||||
```
|
||||
|
||||
## Register a resource
|
||||
|
||||
Create a resource file under `resources/` (example YAML) and register it with the CLI or API.
|
||||
|
||||
## Plan and apply
|
||||
|
||||
```bash
|
||||
# Create a plan using an action on your project
|
||||
cleveragents plan use <action-name> --project my-project
|
||||
# List plans to find the plan ID
|
||||
cleveragents plan list
|
||||
# Review the plan, then apply it by plan ID
|
||||
cleveragents plan apply <plan-id>
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If you encounter issues running the examples above, consult `docs/development/testing.md` and the project README for local development tips.
|
||||
@@ -57,35 +57,32 @@ inherits. Represents a generic container execution environment.
|
||||
| Child types | (none) |
|
||||
| Handler | `DevcontainerHandler` |
|
||||
|
||||
## Auto-Discovery (Planned)
|
||||
## Auto-Discovery
|
||||
|
||||
> **Not yet wired (F31/F23):** The discovery module
|
||||
> (`discover_devcontainers()`) exists and is tested in isolation, but
|
||||
> is **not** invoked during `project link-resource` or any other
|
||||
> production code path. Auto-discovery will be wired in a follow-up PR.
|
||||
> For now, devcontainer-instance resources must be added manually via
|
||||
> `agents resource add`.
|
||||
Auto-discovery is wired into `GitCheckoutHandler.discover_children()` and
|
||||
`FsDirectoryHandler.discover_children()` (issue #4740). When either handler
|
||||
scans a resource location, it calls `discover_devcontainers()` after
|
||||
discovering `fs-directory` children.
|
||||
|
||||
When wired, linking a `git-checkout` or `fs-directory` resource to a
|
||||
project will trigger an auto-discovery hook that scans for devcontainer
|
||||
configurations in the following locations (relative to the resource
|
||||
root):
|
||||
Devcontainer configurations are detected at the following locations
|
||||
(relative to the resource root):
|
||||
|
||||
1. `.devcontainer/devcontainer.json`
|
||||
2. `.devcontainer.json` (root-level)
|
||||
3. `.devcontainer/<name>/devcontainer.json` (named configurations)
|
||||
|
||||
### Discovery Process (Planned)
|
||||
### Discovery Process
|
||||
|
||||
1. **Trigger**: Linking a `git-checkout` or `fs-directory` resource.
|
||||
2. **Scan**: The discovery module checks for configuration files at the
|
||||
well-known paths listed above.
|
||||
1. **Trigger**: `discover_children()` is called on a `git-checkout` or
|
||||
`fs-directory` resource.
|
||||
2. **Scan**: `discover_devcontainers()` checks for configuration files at
|
||||
the well-known paths listed above.
|
||||
3. **Validate**: Each discovered file is parsed as JSON. Invalid files
|
||||
are skipped with a warning.
|
||||
4. **Register**: For each valid configuration:
|
||||
- A `devcontainer-instance` child resource is created under the
|
||||
parent resource.
|
||||
- A `devcontainer-file` child resource is created under the
|
||||
devcontainer instance, pointing to the JSON file.
|
||||
parent resource with `provisioning_state: discovered`.
|
||||
- Named configurations carry the subdirectory name as `config_name`.
|
||||
|
||||
### Lazy Activation
|
||||
|
||||
@@ -250,7 +247,7 @@ confirmation unless `--yes` (`-y`) is passed.
|
||||
| Registry eviction | Terminal-state trackers (stopped/failed) are evicted when count exceeds 200 (`_MAX_TERMINAL_TRACKERS`). Eviction runs automatically after bulk cleanup via `stop_all_active_containers`. Long-running processes with many container cycles may lose historical tracker data. | Acceptable for MVP; persisted state in M7+ eliminates this. |
|
||||
| Sandbox strategy (F22/F25) | Specification uses `container_snapshot`; `SandboxFactory` does not yet implement `snapshot`. Handler now uses `SandboxStrategy.NONE` — the container itself provides isolation. | Implement `container_snapshot` in `SandboxFactory` and switch handler back to it. |
|
||||
| Hardcoded `docker stop` (F21) | `stop_container()` calls `docker stop` directly. Devcontainer CLI can target Podman or other engines; the devcontainer CLI does not yet offer a `stop` subcommand. | Detect the container engine from resource properties or devcontainer CLI config and dispatch to the appropriate stop command. |
|
||||
| Auto-discovery not wired (F23) | Documentation describes auto-discovery on `project link-resource`, but no code path invokes `discover_devcontainers()` during resource linking. | Wire the discovery hook into the resource linking flow. |
|
||||
| Auto-discovery wired (F23 resolved) | `discover_devcontainers()` is now called from `GitCheckoutHandler.discover_children()` and `FsDirectoryHandler.discover_children()`. Devcontainer configs are auto-detected when `discover_children()` is invoked. | Completed in issue #4740. |
|
||||
| Container execution stubbed (F24) | `ToolRunner` returns an error for `ExecutionEnvironment.CONTAINER`, so lazy activation cannot trigger via tool use in production. Lazy activation currently works through `DevcontainerHandler.resolve()` on plan sandbox resolution. | Full container execution support is scoped for a follow-up PR (#616). |
|
||||
|
||||
## DevcontainerHandler Protocol Methods
|
||||
|
||||
+20
-5
@@ -277,7 +277,7 @@ The following standards are integrated into the architecture:
|
||||
[(<span style="color: magenta;"><span style="color: cyan;">--temperature</span>|<span style="color: yellow;">-t</span></span>) <span style="color: #66cc66;"><TEMP></span>] [<span style="color: cyan;">--allow-rxpy-in-run-mode</span>]
|
||||
[<span style="color: cyan;">--skill</span> <span style="color: #66cc66;"><SKILL></span>]... <span style="color: #66cc66;"><NAME></span> <span style="color: #66cc66;"><PROMPT></span>
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor add <span style="color: cyan;">--config</span>|<span style="color: yellow;">-c</span> <span style="color: #66cc66;"><FILE></span> [<span style="color: cyan;">--update</span>]
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor remove <span style="color: #66cc66;"><NAME></span>
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor remove [<span style="color: cyan;">--format</span> <span style="color: #66cc66;"><FORMAT></span>] <span style="color: #66cc66;"><NAME></span>
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor list
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor show <span style="color: #66cc66;"><NAME></span>
|
||||
<span style="color: cyan; font-weight: 600;">agents</span> actor context remove [<span style="color: cyan;">--yes</span>|<span style="color: yellow;">-y</span>] (<span style="color: magenta;"><span style="color: cyan;">--all</span>|<span style="color: yellow;">-a</span>|<span style="color: #66cc66;"><NAME></span></span>)
|
||||
@@ -20019,6 +20019,18 @@ Apply should perform (configurable) validations before committing:
|
||||
|
||||
Validation runs during **Execute**, not during Apply. By the time a plan reaches the Apply phase, all required validations have already passed. Apply commits the sandbox changes to the real resources and does not re-run validations.
|
||||
|
||||
!!! danger "Security Invariant — Empty Validation Summary Blocks Apply"
|
||||
Apply **requires** that at least one required validation was executed during the Execute phase. If no validations were run — because no validations are attached to the plan's resources, or because the validation collection step produced an empty set — the apply gate **blocks** and the plan cannot proceed to Apply.
|
||||
|
||||
This is an intentional security invariant (introduced in PR #7786, fixing issue #7508): apply cannot proceed without at least one validation having run. A plan with no validation attachments is treated the same as a plan with failing validations — it cannot be applied.
|
||||
|
||||
**Blocking conditions for the apply gate**:
|
||||
- Any required validation returned `passed: false` (validation failure)
|
||||
- No required validations were executed at all (empty validation summary)
|
||||
- A plan has no validation attachments (no validations configured)
|
||||
|
||||
The `ApplyValidationSummary.all_required_passed` property returns `True` **only when** at least one required validation was executed AND none of the required validations failed. An empty summary always yields `False`.
|
||||
|
||||
For full details on validation — including the Validation type system, modes, attachment scoping, failure handling, fix-then-revalidate loops, and data model — see the **Validation** section under Core Concepts.
|
||||
|
||||
#### Apply Data Model
|
||||
@@ -22757,7 +22769,8 @@ Validation is the **final step** of the Execute phase. The execution actor's wor
|
||||
2. **Collect applicable validations**: The system resolves all validations from the resource-direct, project, and plan scopes (as described in Attachment Resolution above).
|
||||
3. **Execute validations**: Each applicable validation is invoked as a standard tool call. The validation reads the sandbox state and returns its structured result. Validations may be run in parallel since they are read-only and cannot interfere with each other.
|
||||
4. **Process results**:
|
||||
- **All required validations pass**: Execution is complete. The plan transitions to the review/Apply phase.
|
||||
- **All required validations pass (and at least one ran)**: Execution is complete. The plan transitions to the review/Apply phase.
|
||||
- **No required validations were executed (empty summary)**: The apply gate is blocked. This is treated equivalently to a required validation failure — the plan cannot proceed to Apply. This condition arises when no validations are attached to the plan's resources. See the security invariant note in [Validation in Apply](#validation-in-apply).
|
||||
- **Any required validation fails**: The execution actor enters the fix-then-revalidate loop (see Validation Failure Handling below).
|
||||
- **Informational validations fail**: Results are recorded in the plan's validation summary. No fix attempts are made.
|
||||
|
||||
@@ -23076,7 +23089,7 @@ The validation-related fields stored in the plan data model:
|
||||
| Field | Location | Description |
|
||||
|-------|----------|-------------|
|
||||
| `validation_summary` | `plan.execution` | Array of validation results collected during Execute. Each entry includes the validation name, mode, `passed`, `message`, `data`, execution duration, and attempt number. |
|
||||
| `final_validation_results` | `plan.apply` | Snapshot of the final validation state at the time execution completed (all required validations passing). Identical to the last state of `validation_summary` but stored separately for quick reference. |
|
||||
| `final_validation_results` | `plan.apply` | Snapshot of the final validation state at the time execution completed (all required validations passing, and at least one required validation was executed). Identical to the last state of `validation_summary` but stored separately for quick reference. A plan can only reach the Apply phase if this snapshot is non-empty and all required entries have `passed: true`. |
|
||||
| `validation_attempts` | `plan.execution` | Total number of validation invocations across all fix-then-revalidate loops. Useful for understanding how much effort was spent on validation remediation. |
|
||||
| `validation_fix_history` | `plan.execution` | Per-validation log of fix attempts. Each entry records: the validation that failed, the fix the actor applied, and whether the subsequent re-validation passed. Enables auditing of the fix-then-revalidate process. |
|
||||
|
||||
@@ -45871,6 +45884,8 @@ The relational database follows a normalized design with foreign key constraints
|
||||
|
||||
4. **Optimistic concurrency control**: The `updated_at` timestamp column on mutable entities serves as a version check. Update operations include `WHERE updated_at = <previous_value>` to detect concurrent modifications. On conflict, the operation fails with an explicit error rather than silently overwriting.
|
||||
|
||||
6. **PlanResult Domain Model — `result_success` column** (migration `m9_003_plan_result_success_column`): The legacy `plans` table includes a dedicated `result_success BOOLEAN NULLABLE` column that explicitly records the success status of the result (apply) phase. `PlanRepository._to_domain` derives `PlanResult.success` from this column rather than from `error_message is None`. The `error_message` column is shared between the build phase and the result phase; using it to infer result success would incorrectly mark plans as failed when a build-phase error was recorded but the apply phase later succeeded. When `result_success` is `NULL` (pre-migration records), the repository falls back to the legacy `error_message is None` heuristic for backward compatibility.
|
||||
|
||||
5. **Datetime handling contract**: All timestamps stored in the database are UTC ISO 8601 strings (format: `YYYY-MM-DDTHH:MM:SS.ffffff`). All Python datetime comparisons involving stored timestamps MUST parse the stored string back to a timezone-aware `datetime` object before comparing — **never** compare ISO timestamp strings lexicographically. String comparison is incorrect because different timezone offset formats (`+00:00` vs `Z` vs `+05:30`) produce strings that are not lexicographically comparable even when representing the same moment. The canonical pattern is:
|
||||
|
||||
```python
|
||||
@@ -47121,7 +47136,7 @@ This section defines the ordered milestone plan for CleverAgents v3.x, mapping a
|
||||
| 6 | Hierarchical decomposition creates 4+ levels of subplans | §Subplan Architecture — Hierarchy | `plan tree` shows 4+ nesting levels for complex tasks |
|
||||
| 7 | Parallel execution scales to 10+ concurrent subplans | §Subplan Execution — Parallel | 10 subplans execute concurrently without deadlock or resource starvation |
|
||||
| 8 | Decision correction recomputes only affected subtree | §Correction Model — Selective Recomputation | Unaffected decisions preserved; only downstream decisions recomputed |
|
||||
| 9 | Validation-gated apply: required validations must pass before apply | §Validation Abstraction — Apply Gating | Apply blocked when required validation fails; informational validation does not block |
|
||||
| 9 | Validation-gated apply: required validations must pass before apply | §Validation Abstraction — Apply Gating | Apply blocked when required validation fails; apply also blocked when no validations were run (empty summary); informational validation does not block |
|
||||
| 10 | A realistic porting task completes autonomously | §Autonomy Acceptance | End-to-end task (e.g., port a Python module) completes without human intervention |
|
||||
| 11 | `agents automation-profile add/list/show` functional | §CLI Commands — automation-profile | Round-trip: add profile, list shows it, show displays details |
|
||||
| 12 | `agents plan guard` shows active guards for a plan | §CLI Commands — plan guard | Command shows denylist, budget caps, tool limits |
|
||||
@@ -47133,7 +47148,7 @@ This section defines the ordered milestone plan for CleverAgents v3.x, mapping a
|
||||
- **Guard evaluation order**: Denylist checked first (fast reject), then budget caps, then tool call limits.
|
||||
- **Autonomy threshold**: `0.0` = always automatic; `1.0` = always manual; eight built-in profiles from `manual` to `full-auto`.
|
||||
- **Subtree recomputation**: Only decisions with `depends_on` edges to the corrected decision are recomputed; others preserved.
|
||||
- **Validation gating**: `required` validations block apply; `informational` validations log but do not block.
|
||||
- **Validation gating**: `required` validations block apply; an empty validation summary (no validations ran) also blocks apply; `informational` validations log but do not block.
|
||||
|
||||
#### Definition of Done
|
||||
|
||||
|
||||
@@ -152,6 +152,13 @@ end note
|
||||
| 2026-04-10 | D100 | — | v3.5.0 M6 | 100% | 16.9% | -83.1% | CRITICAL | Day 100: 210/1242 closed |
|
||||
| 2026-04-10 | D100 | — | v3.6.0 M7 | 100% | 33.9% | -66.1% | CRITICAL | Day 100: 152/448 closed |
|
||||
| 2026-04-10 | D100 | — | v3.7.0 M8 | — | 43.8% | — | HIGH | Day 100: 427/975 closed |
|
||||
| 2026-04-12 | D101 | — | v3.2.0 M3 | 100% | 27.8% | -72.2% | CRITICAL | Day 101: 258/926 closed, 664 open (+119) |
|
||||
| 2026-04-12 | D101 | — | v3.3.0 M4 | 100% | 47.0% | -53.0% | CRITICAL | Day 101: 108/230 closed, 122 open (+12) |
|
||||
| 2026-04-12 | D101 | — | v3.4.0 M5 | 100% | 40.2% | -59.8% | CRITICAL | Day 101: 137/341 closed, 204 open (+35) |
|
||||
| 2026-04-12 | D101 | — | v3.5.0 M6 | 100% | 17.0% | -83.0% | CRITICAL | Day 101: 201/1178 closed, 977 open (+135) |
|
||||
| 2026-04-12 | D101 | — | v3.6.0 M7 | 100% | 35.2% | -64.8% | CRITICAL | Day 101: 152/432 closed, 280 open (+48) |
|
||||
| 2026-04-12 | D101 | — | v3.7.0 M8 | — | 44.8% | — | HIGH | Day 101: 427/953 closed, 526 open (+28) |
|
||||
| 2026-04-12 | D101 | — | v3.8.0 M9 | — | 27.0% | — | HIGH | Day 101: 132/489 closed, 357 open (+29) |
|
||||
| 2026-04-13 | D103 | C2 | v3.2.0 M3 | 100% | 25.7% | -74.3% | CRITICAL | Cycle 2: 269/1045 closed, 776 open |
|
||||
| 2026-04-13 | D103 | C2 | v3.3.0 M4 | 100% | 42.4% | -57.6% | CRITICAL | Cycle 2: 109/257 closed, 148 open |
|
||||
| 2026-04-13 | D103 | C2 | v3.4.0 M5 | 100% | 37.4% | -62.6% | CRITICAL | Cycle 2: 139/372 closed, 233 open |
|
||||
|
||||
@@ -9,9 +9,10 @@ Feature: ACMS Index Data Model and File Traversal Engine
|
||||
|
||||
Scenario: Create an index entry with file metadata
|
||||
When I create an index entry with:
|
||||
| path | /project/src/main.py |
|
||||
| file_type | python |
|
||||
| size_bytes | 1024 |
|
||||
| key | value |
|
||||
| path | /project/src/main.py |
|
||||
| file_type | python |
|
||||
| size_bytes | 1024 |
|
||||
Then the index entry should have path "/project/src/main.py"
|
||||
And the index entry should have file type "python"
|
||||
And the index entry should have size 1024 bytes
|
||||
@@ -116,7 +117,7 @@ Feature: ACMS Index Data Model and File Traversal Engine
|
||||
Scenario: Get entry count from index
|
||||
Given I have an index with 10 entries
|
||||
When I get the entry count
|
||||
Then the count should be 10
|
||||
Then idx the index count should be 10
|
||||
|
||||
Scenario: Remove entry from index
|
||||
Given I have an index with 3 entries
|
||||
@@ -131,8 +132,9 @@ Feature: ACMS Index Data Model and File Traversal Engine
|
||||
| /project/tests/test_main.py | python | test | cold |
|
||||
| /project/docs/readme.md | markdown | docs | cold |
|
||||
When I query the index with filters:
|
||||
| path_pattern | src |
|
||||
| file_type | python |
|
||||
| tier | hot |
|
||||
| filter | value |
|
||||
| path_pattern | src |
|
||||
| file_type | python |
|
||||
| tier | hot |
|
||||
Then I should get 1 result
|
||||
And the result should have path "/project/src/main.py"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
Feature: agents actor add NAME positional argument
|
||||
As a user of the CleverAgents CLI
|
||||
I want to pass the actor name as a positional argument to `agents actor add`
|
||||
So that the CLI matches the spec synopsis: agents actor add <NAME> --config <FILE>
|
||||
So that the CLI matches the spec synopsis: agents actor add --config <FILE> [<NAME>]
|
||||
|
||||
@tdd_issue @tdd_issue_4230 @tdd_expected_fail
|
||||
Scenario: actor add accepts NAME as positional argument
|
||||
@@ -17,8 +17,16 @@ Feature: agents actor add NAME positional argument
|
||||
When I run actor add with NAME positional argument overriding config name
|
||||
Then the actor add should use the positional NAME not the config name
|
||||
|
||||
Scenario: actor add without NAME positional argument fails
|
||||
@tdd_issue @tdd_issue_4186
|
||||
Scenario: actor add without NAME positional argument uses config name
|
||||
Given an actor CLI runner
|
||||
And I have an actor JSON config file with name "local/config-derived-actor"
|
||||
When I run actor add with config but no NAME positional argument
|
||||
Then the actor add should succeed using the config name
|
||||
|
||||
@tdd_issue @tdd_issue_4186
|
||||
Scenario: actor add without NAME and without config name field raises BadParameter
|
||||
Given an actor CLI runner
|
||||
And I have an actor JSON config file without a name field
|
||||
When I run actor add with config but no NAME positional argument
|
||||
Then the actor command should fail with missing argument error
|
||||
Then the actor add should fail with a BadParameter error about missing actor name
|
||||
|
||||
@@ -5,7 +5,7 @@ Feature: agents actor add enforces --update flag for existing actors
|
||||
I want `agents actor add` to fail with a clear error when re-adding an existing actor
|
||||
So that I cannot accidentally overwrite actor configurations without explicit intent
|
||||
|
||||
@tdd_issue @tdd_issue_2609 @tdd_expected_fail @tdd_issue_4178
|
||||
@tdd_issue @tdd_issue_2609 @tdd_issue_4178
|
||||
Scenario: Re-adding an existing actor without --update fails with error panel
|
||||
Given an actor add CLI runner where the actor already exists
|
||||
When I run actor add without the --update flag
|
||||
@@ -14,7 +14,7 @@ Feature: agents actor add enforces --update flag for existing actors
|
||||
And the actor-add-enforcement output should contain "Use --update to replace the existing actor definition"
|
||||
And the actor-add-enforcement output should contain the registration timestamp
|
||||
|
||||
@tdd_issue @tdd_issue_2609 @tdd_expected_fail @tdd_issue_4178
|
||||
@tdd_issue @tdd_issue_2609 @tdd_issue_4178
|
||||
Scenario: Re-adding an existing actor without --update shows error status line
|
||||
Given an actor add CLI runner where the actor already exists
|
||||
When I run actor add without the --update flag
|
||||
@@ -22,14 +22,14 @@ Feature: agents actor add enforces --update flag for existing actors
|
||||
And the actor-add-enforcement output should contain "Actor already registered"
|
||||
And the actor-add-enforcement output should contain "use --update to replace"
|
||||
|
||||
@tdd_issue @tdd_issue_2609 @tdd_expected_fail @tdd_issue_4178
|
||||
@tdd_issue @tdd_issue_2609 @tdd_issue_4178
|
||||
Scenario: Re-adding an existing actor with --update succeeds
|
||||
Given an actor add CLI runner where the actor already exists
|
||||
When I run actor add with the --update flag
|
||||
Then the actor-add-enforcement exit code should be 0
|
||||
And the actor-add-enforcement output should contain "Actor updated"
|
||||
|
||||
@tdd_issue @tdd_issue_2609 @tdd_expected_fail @tdd_issue_4178
|
||||
@tdd_issue @tdd_issue_2609 @tdd_issue_4178
|
||||
Scenario: Adding a new actor without --update succeeds
|
||||
Given an actor add CLI runner where the actor does not exist
|
||||
When I run actor add without the --update flag
|
||||
|
||||
@@ -59,6 +59,11 @@ Feature: Actor CLI YAML-first alignment
|
||||
When I run actor remove with namespaced name
|
||||
Then the actor remove should succeed for namespaced name
|
||||
|
||||
Scenario: Actor remove outputs JSON format
|
||||
Given an actor CLI runner
|
||||
When I run actor remove with format json
|
||||
Then the actor remove output should be valid JSON envelope
|
||||
|
||||
Scenario: Actor update outputs JSON format
|
||||
Given an actor CLI runner
|
||||
When I run actor update with format json
|
||||
|
||||
@@ -279,140 +279,10 @@ Feature: Actor configuration uncovered lines coverage
|
||||
When I build an actor configuration from blob {"provider": "only-provider"}
|
||||
Then a ValueError should be raised containing "model is required"
|
||||
|
||||
Scenario: from_blob merges default and v2 and override options
|
||||
When I build an actor configuration from structured blob with defaults and overrides:
|
||||
"""
|
||||
{
|
||||
"blob": {
|
||||
"provider": "cli-provider",
|
||||
"model": "cli-model",
|
||||
"options": {"user": "blob"},
|
||||
"agents": {
|
||||
"writer": {
|
||||
"config": {
|
||||
"options": {"v2": "v2-option"}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"default_options": {"base": "default"},
|
||||
"option_overrides": {"user": "override", "extra": "added"}
|
||||
}
|
||||
"""
|
||||
Then the actor configuration should have provider "cli-provider" and model "cli-model"
|
||||
And the actor configuration options should equal {"base": "default", "v2": "v2-option", "user": "override", "extra": "added"}
|
||||
|
||||
Scenario: from_blob with None blob defaults to empty data
|
||||
When I build actor config from None blob with provider and model overrides
|
||||
Then the actor configuration should have provider "override-provider" and model "override-model"
|
||||
|
||||
# --- _extract_v2_actor: lines 275-307 ---
|
||||
|
||||
Scenario: v2 YAML actor config infers provider and model
|
||||
Given an actor config file "v2.yaml" with content:
|
||||
"""
|
||||
cleveragents:
|
||||
default_router: main_router
|
||||
agents:
|
||||
paper_writer:
|
||||
type: llm
|
||||
config:
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
unsafe: true
|
||||
options:
|
||||
temperature: 0.5
|
||||
routes:
|
||||
main_router:
|
||||
type: stream
|
||||
operators:
|
||||
- type: map
|
||||
params:
|
||||
agent: paper_writer
|
||||
publications:
|
||||
- __output__
|
||||
"""
|
||||
When I parse the actor configuration from file "v2.yaml" with overrides:
|
||||
"""
|
||||
{}
|
||||
"""
|
||||
Then the actor configuration should have provider "openai" and model "gpt-4"
|
||||
And the actor configuration unsafe flag should be true
|
||||
And the actor configuration graph descriptor should include key "routes"
|
||||
And the actor configuration graph descriptor should include key "agents"
|
||||
And the actor configuration graph descriptor should include key "cleveragents"
|
||||
And the actor configuration options should equal {"temperature": 0.5}
|
||||
|
||||
Scenario: v2 extraction includes merges and templates keys when present
|
||||
Given an actor config file "v2_merges.yaml" with content:
|
||||
"""
|
||||
agents:
|
||||
writer:
|
||||
config:
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
merges:
|
||||
combined:
|
||||
type: join
|
||||
templates:
|
||||
default:
|
||||
system_prompt: "hello"
|
||||
"""
|
||||
When I parse the actor configuration from file "v2_merges.yaml" with overrides:
|
||||
"""
|
||||
{}
|
||||
"""
|
||||
Then the actor configuration should have provider "openai" and model "gpt-4"
|
||||
And the actor configuration graph descriptor should include key "merges"
|
||||
And the actor configuration graph descriptor should include key "templates"
|
||||
|
||||
Scenario: v2 extraction handles agent entry without config block
|
||||
Given an actor config file "v2_no_config.yaml" with content:
|
||||
"""
|
||||
provider: fallback-provider
|
||||
model: fallback-model
|
||||
agents:
|
||||
first:
|
||||
type: llm
|
||||
"""
|
||||
When I parse the actor configuration from file "v2_no_config.yaml" with overrides:
|
||||
"""
|
||||
{}
|
||||
"""
|
||||
Then the actor configuration should have provider "fallback-provider" and model "fallback-model"
|
||||
|
||||
# --- _extract_v2_options: lines 316-326 ---
|
||||
|
||||
Scenario: v2 extraction skips non-dict agent entries
|
||||
Given an actor config file "invalid_v2.yaml" with content:
|
||||
"""
|
||||
provider: fallback-provider
|
||||
model: fallback-model
|
||||
agents:
|
||||
first: not-a-dict
|
||||
"""
|
||||
When I parse the actor configuration from file "invalid_v2.yaml" with overrides:
|
||||
"""
|
||||
{}
|
||||
"""
|
||||
Then the actor configuration should have provider "fallback-provider" and model "fallback-model"
|
||||
|
||||
Scenario: v2 options extraction returns None for agents without options
|
||||
Given an actor config file "v2_no_opts.yaml" with content:
|
||||
"""
|
||||
agents:
|
||||
writer:
|
||||
config:
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
When I parse the actor configuration from file "v2_no_opts.yaml" with overrides:
|
||||
"""
|
||||
{}
|
||||
"""
|
||||
Then the actor configuration should have provider "openai" and model "gpt-4"
|
||||
And the actor configuration options should equal {}
|
||||
|
||||
Scenario: from_blob reads provider_type and model_id aliases
|
||||
When I build an actor configuration from blob {"provider_type": "aliased-provider", "model_id": "aliased-model"}
|
||||
Then the actor configuration should have provider "aliased-provider" and model "aliased-model"
|
||||
|
||||
@@ -15,11 +15,19 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
And the registered actor name should be "local/my-assistant"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
@tdd_issue @tdd_issue_4300
|
||||
Scenario: registry.add() accepts spec-compliant actors: map with separate provider and model
|
||||
When I add a spec-compliant YAML with actors map and separate provider model
|
||||
Then the actor should be registered with provider "anthropic" and model "claude-3"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
@tdd_issue @tdd_issue_4321 @tdd_issue_4300
|
||||
Scenario: registry.add() with nested-only type extracts provider/model from nested config
|
||||
When I add a YAML with type only in nested actors map and provider/model in nested config
|
||||
Then the actor should be registered with provider "custom" and model "fpga-model"
|
||||
And the registered actor name should be "local/fpga-strategist"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── actors: map with unsafe flag ───────────────────────────────────
|
||||
|
||||
Scenario: registry.add() preserves unsafe flag from nested spec-compliant config
|
||||
@@ -53,24 +61,13 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
Then the actor should be registered with provider "nested-provider" and model "top-level-model"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── Graph descriptor preserved from nested config ──────────────────
|
||||
|
||||
Scenario: registry.add() preserves graph descriptor from nested spec-compliant config
|
||||
When I add a spec-compliant YAML with actors map and combined actor field
|
||||
Then the registered actor graph descriptor should contain key "actors"
|
||||
|
||||
# ── Top-level provider+model still picks up nested unsafe/graph ──────
|
||||
# ── Top-level provider+model still picks up nested unsafe ───────────
|
||||
|
||||
Scenario: registry.add() with top-level provider and model detects nested unsafe flag
|
||||
When I add a YAML with top-level provider and model and nested actors map with unsafe flag
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should be marked unsafe
|
||||
|
||||
Scenario: registry.add() with top-level provider and model detects nested graph descriptor
|
||||
When I add a YAML with top-level provider and model and nested actors map with graph descriptor
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor graph descriptor should contain key "actors"
|
||||
|
||||
# ── Unsafe confirmation gate ───────────────────────────────────────
|
||||
|
||||
Scenario: registry.add() rejects unsafe actor without confirmation
|
||||
@@ -94,12 +91,6 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should not be marked unsafe
|
||||
|
||||
# ── Missing provider/model still rejected for non-v3 YAML ─────────
|
||||
|
||||
Scenario: registry.add() rejects non-v3 YAML without any provider or model
|
||||
When I attempt to add a YAML with no provider or model anywhere
|
||||
Then a spec-yaml ValidationError should be raised containing "provider"
|
||||
|
||||
# ── update=True path ───────────────────────────────────────────────
|
||||
|
||||
Scenario: registry.add() with update=True overwrites an existing actor using spec-compliant YAML
|
||||
@@ -139,366 +130,6 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
And the registered actor should be marked unsafe
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── Multi-actor YAML rejection ───────────────────────────────────
|
||||
|
||||
Scenario: registry.add() rejects multi-actor YAML with a ValidationError
|
||||
When I attempt to add a multi-actor YAML with two actor entries
|
||||
Then a spec-yaml ValidationError should be raised containing "single-actor"
|
||||
|
||||
Scenario: registry.add() rejects multi-actor YAML via agents: fallback when actors: is null
|
||||
When I attempt to add a YAML with actors null and multi-entry agents map
|
||||
Then a spec-yaml ValidationError should be raised containing "single-actor"
|
||||
|
||||
# ── _extract_v2_actor handles actors: key ──────────────────────────
|
||||
|
||||
Scenario: _extract_v2_actor extracts provider/model from actors: map
|
||||
When I call _extract_v2_actor with an actors map containing combined actor field
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: _extract_v2_actor extracts from actors: map with separate fields
|
||||
When I call _extract_v2_actor with an actors map containing separate provider model
|
||||
Then the spec-yaml extracted provider should be "anthropic"
|
||||
And the spec-yaml extracted model should be "claude-3"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: _extract_v2_actor prefers actors: key over agents: key
|
||||
When I call _extract_v2_actor with both actors and agents maps
|
||||
Then the spec-yaml extracted provider should be "actors-provider"
|
||||
And the spec-yaml extracted model should be "actors-model"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: _extract_v2_actor graph descriptor contains agent key matching the actor entry name
|
||||
When I call _extract_v2_actor with an actors map containing combined actor field
|
||||
Then the spec-yaml extracted graph descriptor should contain key "agent"
|
||||
And the spec-yaml extracted graph descriptor agent value should be "my_assistant"
|
||||
|
||||
Scenario: _extract_v2_actor with actors map containing unsafe flag
|
||||
When I call _extract_v2_actor with an actors map containing unsafe true
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted unsafe flag should be True
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: _extract_v2_actor returns None for empty data
|
||||
When I call _extract_v2_actor with an empty dict
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
|
||||
Scenario: _extract_v2_actor returns None for actors key with empty map
|
||||
When I call _extract_v2_actor with actors key containing empty map
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
|
||||
Scenario: _extract_v2_actor returns None for actors key with None value
|
||||
When I call _extract_v2_actor with actors key containing None value
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
|
||||
# ── _extract_v2_actor handles agents: key ─────────────────────────
|
||||
|
||||
Scenario: _extract_v2_actor returns graph descriptor with agents map_key
|
||||
When I call _extract_v2_actor with an agents map containing combined actor field
|
||||
Then the spec-yaml extracted graph descriptor should contain key "agents"
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
# ── _extract_v2_actor edge cases: non-dict / missing config ────────
|
||||
|
||||
Scenario: _extract_v2_actor returns None for non-dict first entry
|
||||
When I call _extract_v2_actor with a non-dict first entry in actors map
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
Scenario: _extract_v2_actor returns None for dict entry missing config block
|
||||
When I call _extract_v2_actor with a dict entry missing config block
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
# ── _extract_v2_actor edge case: actors: {} blocks agents: fallback ─
|
||||
|
||||
Scenario: _extract_v2_actor with empty actors dict blocks agents fallback
|
||||
When I call _extract_v2_actor with empty actors dict and valid agents map
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
# ── _extract_v2_actor edge case: actors: [] (list type) ────────────
|
||||
|
||||
Scenario: _extract_v2_actor with actors as list returns None
|
||||
When I call _extract_v2_actor with actors as a list
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should be None
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
# ── _extract_v2_options handles actors: and agents: keys ───────────
|
||||
|
||||
Scenario: _extract_v2_options extracts options from actors: map
|
||||
When I call _extract_v2_options with an actors map containing options
|
||||
Then the spec-yaml extracted options should contain key "temperature" with value 0.7
|
||||
|
||||
Scenario: _extract_v2_options extracts options from agents: map
|
||||
When I call _extract_v2_options with an agents map containing options
|
||||
Then the spec-yaml extracted options should contain key "max_tokens" with value 1024
|
||||
|
||||
Scenario: _extract_v2_options returns None for actors key with empty map
|
||||
When I call _extract_v2_options with actors key containing empty map
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options returns None for actors key with None value
|
||||
When I call _extract_v2_options with actors key containing None value
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options returns None for actors key with list value
|
||||
When I call _extract_v2_options with actors key containing list value
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options returns None when config block has no options key
|
||||
When I call _extract_v2_options with an actors map where config has no options key
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options returns None for non-dict first entry in actors map
|
||||
When I call _extract_v2_options with a non-dict first entry in actors map
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options returns None for dict entry missing config block
|
||||
When I call _extract_v2_options with a dict entry missing config block
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
Scenario: _extract_v2_options prefers actors: key over agents: key
|
||||
When I call _extract_v2_options with both actors and agents maps containing options
|
||||
Then the spec-yaml extracted options should contain key "source" with value "actors"
|
||||
|
||||
# ── Unsafe coercion edge cases (unsafe coercion) ────────────────
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: "no" as False (not truthy string)
|
||||
When I call _extract_v2_actor with unsafe value "no"
|
||||
Then the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: "yes" as False (not truthy string)
|
||||
When I call _extract_v2_actor with unsafe value "yes"
|
||||
Then the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: 1 (integer) as True
|
||||
When I call _extract_v2_actor with unsafe value 1
|
||||
Then the spec-yaml extracted unsafe flag should be True
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
Scenario: registry.add() accepts actors map with unsafe: 1 (integer) and unsafe flag
|
||||
When I add a YAML with actors map where unsafe is integer 1 and the unsafe flag set
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should be marked unsafe
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: 1.0 (float) as True
|
||||
When I call _extract_v2_actor with unsafe value 1.0
|
||||
Then the spec-yaml extracted unsafe flag should be True
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: 2 (integer > 1) as False
|
||||
When I call _extract_v2_actor with unsafe value 2
|
||||
Then the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
Scenario: _extract_v2_actor treats unsafe: 0 (integer zero) as False
|
||||
When I call _extract_v2_actor with unsafe value 0
|
||||
Then the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
|
||||
# ── Top-level unsafe: 1 (integer) through registry.add() ────────────
|
||||
|
||||
Scenario: registry.add() rejects top-level unsafe: 1 (integer) without confirmation
|
||||
When I attempt to add a YAML with top-level unsafe integer 1 and no flag
|
||||
Then a spec-yaml ValidationError should be raised containing "unsafe"
|
||||
|
||||
Scenario: registry.add() accepts top-level unsafe: 1 (integer) with unsafe flag
|
||||
When I add a YAML with top-level unsafe integer 1 and the unsafe flag set
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should be marked unsafe
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── Top-level graph_descriptor key through registry.add() (T7) ──────
|
||||
|
||||
Scenario: registry.add() resolves graph descriptor from top-level graph_descriptor key
|
||||
When I add a YAML with top-level provider model and graph_descriptor key
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor graph descriptor should contain key "workflow"
|
||||
|
||||
Scenario: _extract_v2_actor includes top-level routes key in graph descriptor
|
||||
When I call _extract_v2_actor with an actors map and a top-level routes key
|
||||
Then the spec-yaml extracted graph descriptor should contain key "routes"
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
# ── Legacy graph key fallback (M3) ─────────────────────────────────
|
||||
|
||||
Scenario: registry.add() resolves graph descriptor from legacy top-level graph key
|
||||
When I add a YAML with top-level provider model and legacy graph key
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor graph descriptor should contain key "workflow"
|
||||
|
||||
# ── Empty actors map through registry.add() (M4) ───────────────────
|
||||
|
||||
Scenario: registry.add() with empty actors map does not fall back to agents map for provider/model
|
||||
When I attempt to add a YAML with empty actors map and valid agents map but no top-level provider
|
||||
Then a spec-yaml ValidationError should be raised containing "provider"
|
||||
|
||||
# ── provider_type / model_id aliases in nested config (m1) ─────────
|
||||
|
||||
Scenario: _extract_v2_actor extracts provider from provider_type alias in nested config
|
||||
When I call _extract_v2_actor with an actors map using provider_type alias
|
||||
Then the spec-yaml extracted provider should be "alias-provider"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: _extract_v2_actor extracts model from model_id alias in nested config
|
||||
When I call _extract_v2_actor with an actors map using model_id alias
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "alias-model"
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
# ── _extract_v2_options with empty dict (m4) ────────────────────────
|
||||
|
||||
Scenario: _extract_v2_options returns None for empty dict input
|
||||
When I call _extract_v2_options with an empty dict
|
||||
Then the spec-yaml extracted options should be None
|
||||
|
||||
# ── compiled_metadata value assertion (m5) ──────────────────────────
|
||||
|
||||
Scenario: registry.add() forwards compiled_metadata with correct values
|
||||
When I add a spec-compliant YAML with schema_version "2.0" and compiled_metadata
|
||||
Then the registered actor compiled metadata key "key" should have value "val"
|
||||
|
||||
# ── Combined actor field edge cases ────────────────────────────────
|
||||
|
||||
Scenario: Combined actor field without slash is ignored
|
||||
When I call _extract_v2_actor with an actors map where actor field has no slash
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
|
||||
Scenario: Combined actor field does not override explicit provider but fills missing model
|
||||
When I call _extract_v2_actor with an actors map where both actor and provider exist
|
||||
Then the spec-yaml extracted provider should be "explicit-provider"
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: Combined actor field does not override explicit model but fills missing provider
|
||||
When I call _extract_v2_actor with an actors map where both actor and model exist
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "explicit-model"
|
||||
And the spec-yaml extracted unsafe flag should be False
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
# ── Combined actor field malformed input edge cases ────────────────
|
||||
|
||||
Scenario: Combined actor field with empty provider part yields no provider
|
||||
When I call _extract_v2_actor with an actors map where actor field has empty provider
|
||||
Then the spec-yaml extracted provider should be None
|
||||
And the spec-yaml extracted model should be "gpt-4"
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
Scenario: Combined actor field with empty model part yields no model
|
||||
When I call _extract_v2_actor with an actors map where actor field has empty model
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be None
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
# ── Combined actor field with multiple slashes (L1) ────────────────
|
||||
|
||||
Scenario: Combined actor field with multiple slashes splits on first slash only
|
||||
When I call _extract_v2_actor with an actors map where actor field has multiple slashes
|
||||
Then the spec-yaml extracted provider should be "openai"
|
||||
And the spec-yaml extracted model should be "gpt-4/extra"
|
||||
And the spec-yaml extracted graph descriptor should contain key "actors"
|
||||
|
||||
# ── actors: null + valid agents: through registry.add() (L2) ───────
|
||||
|
||||
Scenario: registry.add() with actors: null falls back to valid agents: map
|
||||
When I add a YAML with actors null and valid agents map
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── Top-level provider_type / model_id aliases through registry.add() (T8) ──
|
||||
|
||||
Scenario: registry.add() accepts top-level provider_type alias
|
||||
When I add a YAML with top-level provider_type alias and model
|
||||
Then the actor should be registered with provider "alias-provider" and model "gpt-4"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
Scenario: registry.add() accepts top-level model_id alias
|
||||
When I add a YAML with top-level provider and model_id alias
|
||||
Then the actor should be registered with provider "openai" and model "alias-model"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
# ── Top-level unsafe string coercion through registry.add() (T9) ───
|
||||
|
||||
Scenario: registry.add() treats top-level unsafe: "yes" as not unsafe (no gate rejection)
|
||||
When I add a YAML with top-level unsafe string "yes" and provider model
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should not be marked unsafe
|
||||
|
||||
Scenario: registry.add() treats top-level unsafe: "no" as not unsafe (no gate rejection)
|
||||
When I add a YAML with top-level unsafe string "no" and provider model
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor should not be marked unsafe
|
||||
|
||||
# ── Nested options extraction through registry.add() (M1) ──────────
|
||||
|
||||
Scenario: registry.add() extracts and preserves nested config options
|
||||
When I add a spec-compliant YAML with actors map and nested options
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor config blob should contain options key "temperature" with value 0.9
|
||||
And the registered actor config blob should contain options key "max_tokens" with value 2000
|
||||
|
||||
Scenario: registry.add() merges nested and top-level options (nested base, top-level overrides)
|
||||
When I add a YAML with both top-level and nested options
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor config blob should contain options key "temperature" with value 0.7
|
||||
And the registered actor config blob should contain options key "max_tokens" with value 2000
|
||||
And the registered actor config blob should contain options key "top_p" with value 0.95
|
||||
|
||||
Scenario: registry.add() with non-dict top-level options uses nested options
|
||||
When I add a YAML with non-dict top-level options and nested options
|
||||
Then the actor should be registered with provider "openai" and model "gpt-4"
|
||||
And the registered actor config blob should contain options key "temperature" with value 0.9
|
||||
|
||||
Scenario: registry.add() sets source: "yaml" default in config_blob
|
||||
When I add a spec-compliant YAML with actors map and combined actor field
|
||||
Then the registered actor config blob should contain source "yaml"
|
||||
|
||||
# ── _extract_v2_options shallow copy mutation isolation (NIT-2) ─────
|
||||
|
||||
Scenario: _extract_v2_options returns a shallow copy that does not mutate the original blob
|
||||
When I call _extract_v2_options and mutate the returned dict
|
||||
Then the original blob options should be unmodified
|
||||
|
||||
# ── M4: update=True creates actor when it doesn't exist ────────────
|
||||
|
||||
Scenario: registry.add() with update=True creates actor when it doesn't exist
|
||||
@@ -518,25 +149,5 @@ Feature: ActorRegistry.add() accepts spec-compliant actor YAML formats
|
||||
When upsert_actor is configured to raise RuntimeError and I add a valid YAML
|
||||
Then a RuntimeError should have been propagated from add
|
||||
|
||||
# ── M7: actors: false (boolean) blocks agents fallback ─────────────
|
||||
|
||||
|
||||
Scenario: registry.add() with actors: false blocks agents fallback and raises provider error
|
||||
When I attempt to add a YAML with actors false and valid agents map
|
||||
Then a spec-yaml ValidationError should be raised containing "provider"
|
||||
|
||||
# ── M8: non-dict compiled_metadata causes Pydantic error ───────────
|
||||
|
||||
Scenario: registry.add() with non-dict compiled_metadata raises a Pydantic validation error
|
||||
When I attempt to add a valid YAML with compiled_metadata as a non-dict string
|
||||
Then a Pydantic validation error should be raised for compiled_metadata
|
||||
|
||||
# ── M9: provider: 0 (integer zero) falls through to provider_type ──
|
||||
|
||||
Scenario: registry.add() with provider: 0 falls through to provider_type fallback
|
||||
When I attempt to add a YAML with provider integer 0 and no provider_type
|
||||
Then a spec-yaml ValidationError should be raised containing "provider"
|
||||
|
||||
Scenario: registry.add() with provider: 0 and valid provider_type uses provider_type
|
||||
When I add a YAML with provider integer 0 and a valid provider_type
|
||||
Then the actor should be registered with provider "fallback-provider" and model "gpt-4"
|
||||
And the registered actor should exist in the actor service
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
Feature: Cross-actor subgraph cycle detection reads actor_ref field
|
||||
As a CleverAgents developer
|
||||
I want the actor compiler to correctly detect cross-actor subgraph cycles
|
||||
So that mutually-referencing actors raise SubgraphCycleError at compile time
|
||||
|
||||
Background:
|
||||
Given the actor compiler is available
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Bug fix: actor_ref is a top-level NodeDefinition field
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@tdd_issue @tdd_issue_1431
|
||||
Scenario: Mutually-referencing actors raise SubgraphCycleError
|
||||
Given actor "test/actor-a" has a subgraph node with actor_ref "test/actor-b"
|
||||
And actor "test/actor-b" has a subgraph node with actor_ref "test/actor-a"
|
||||
And a registry containing both actors
|
||||
When I compile actor "test/actor-a" with the registry resolver
|
||||
Then the compilation should raise SubgraphCycleError
|
||||
And the cycle error message should mention "cycle"
|
||||
|
||||
@tdd_issue @tdd_issue_1431
|
||||
Scenario: Non-cyclic subgraph reference compiles successfully
|
||||
Given actor "test/actor-x" has a subgraph node with actor_ref "test/actor-y"
|
||||
And actor "test/actor-y" has no subgraph nodes
|
||||
And a registry containing both actors
|
||||
When I compile actor "test/actor-x" with the registry resolver
|
||||
Then the actor compilation should succeed
|
||||
And the actor subgraph_refs should map "sub" to "test/actor-y"
|
||||
|
||||
@tdd_issue @tdd_issue_1431
|
||||
Scenario: Subgraph node actor_ref is reflected in compiled metadata
|
||||
Given actor "test/actor-p" has a subgraph node with actor_ref "test/actor-q"
|
||||
And actor "test/actor-q" has no subgraph nodes
|
||||
And a registry containing both actors
|
||||
When I compile actor "test/actor-p" with the registry resolver
|
||||
Then the actor compilation should succeed
|
||||
And the compiled node "sub" should have subgraph set to "test/actor-q"
|
||||
@@ -26,12 +26,6 @@ Feature: v3 Actor YAML schema support in CLI and registry
|
||||
When I call ActorConfiguration.from_blob with the v3 blob
|
||||
Then the configuration should have a graph_descriptor with type "graph"
|
||||
|
||||
Scenario: ActorConfiguration.from_blob falls through to v2 for legacy YAML
|
||||
Given a v2 actor blob with agents and routes
|
||||
When I call ActorConfiguration.from_blob with the v2 blob
|
||||
Then the configuration should have provider "openai"
|
||||
And the configuration should have model "gpt-4"
|
||||
|
||||
Scenario: ActorRegistry.add validates v3 LLM actor YAML
|
||||
Given a mock actor service for v3 testing
|
||||
And a v3 LLM actor YAML text
|
||||
@@ -67,11 +61,6 @@ Feature: v3 Actor YAML schema support in CLI and registry
|
||||
And the reactive config should have a graph route
|
||||
And the graph route edges should use source and target keys
|
||||
|
||||
Scenario: v3 format detection returns false for v2 data
|
||||
Given a v2 actor config dict with agents key
|
||||
When I check if the config is v3 format
|
||||
Then it should not be detected as v3
|
||||
|
||||
# M14: test update=True path
|
||||
Scenario: ActorRegistry.add with update=True overwrites existing actor
|
||||
Given a mock actor service for v3 testing
|
||||
@@ -174,27 +163,6 @@ Feature: v3 Actor YAML schema support in CLI and registry
|
||||
When I call ActorRegistry.add with a non-mapping YAML string
|
||||
Then a ValidationError should be raised mentioning mapping
|
||||
|
||||
# Coverage: registry.py — _add_legacy v3 schema validation failure
|
||||
Scenario: ActorRegistry._add_legacy rejects invalid v3 YAML via schema validation
|
||||
Given a mock actor service for v3 testing
|
||||
And a legacy v3 YAML text with invalid schema
|
||||
When I call ActorRegistry.add with the legacy invalid v3 YAML
|
||||
Then a ValidationError should be raised mentioning invalid v3 actor
|
||||
|
||||
# Coverage: registry.py — _add_legacy missing provider/model in non-v3 blob
|
||||
Scenario: ActorRegistry._add_legacy rejects non-v3 blob missing provider and model
|
||||
Given a mock actor service for v3 testing
|
||||
And a legacy non-v3 YAML text missing provider and model
|
||||
When I call ActorRegistry.add with the legacy non-v3 YAML
|
||||
Then a ValidationError should be raised mentioning provider and model
|
||||
|
||||
# Coverage: registry.py — _add_legacy unsafe flag rejection
|
||||
Scenario: ActorRegistry._add_legacy rejects unsafe legacy actor without allow_unsafe
|
||||
Given a mock actor service for v3 testing
|
||||
And a legacy actor YAML text marked unsafe
|
||||
When I call ActorRegistry.add with the legacy unsafe YAML
|
||||
Then a ValidationError should be raised mentioning unsafe
|
||||
|
||||
# Coverage: registry.py — upsert_actor v3 validation failure
|
||||
Scenario: ActorRegistry.upsert_actor rejects invalid v3 config_blob
|
||||
Given a mock actor service for v3 testing
|
||||
|
||||
@@ -34,6 +34,7 @@ Feature: Architecture validation
|
||||
Then all application variables should use CLEVERAGENTS_ prefix
|
||||
And provider variables should keep their original prefix
|
||||
|
||||
@tdd_issue @tdd_issue_4186
|
||||
Scenario: Type hints are used throughout
|
||||
Given the source code exists
|
||||
When I check for type annotations
|
||||
|
||||
@@ -99,3 +99,23 @@ Feature: Behave-parallel conditional log replay
|
||||
Given behave_parallel worker results with one crashed chunk and one passing chunk
|
||||
When behave_parallel I aggregate the worker results
|
||||
Then behave_parallel the aggregated summary should have failures
|
||||
|
||||
# ---- PassSuppressFormatter: per-scenario output suppression ----
|
||||
|
||||
Scenario: PassSuppressFormatter suppresses output for a passing scenario
|
||||
Given behave_parallel a PassSuppressFormatter backed by a captured stream
|
||||
When behave_parallel I simulate a passing scenario through the formatter
|
||||
Then behave_parallel the formatter real stream output should be empty
|
||||
|
||||
Scenario: PassSuppressFormatter emits full output for a failing scenario
|
||||
Given behave_parallel a PassSuppressFormatter backed by a captured stream
|
||||
When behave_parallel I simulate a failing scenario through the formatter
|
||||
Then behave_parallel the formatter real stream output should contain "Failing scenario"
|
||||
And behave_parallel the formatter real stream output should contain "I fail"
|
||||
And behave_parallel the formatter real stream output should contain "AssertionError"
|
||||
|
||||
Scenario: PassSuppressFormatter only shows failed scenarios in a mixed run
|
||||
Given behave_parallel a PassSuppressFormatter backed by a captured stream
|
||||
When behave_parallel I simulate one passing then one failing scenario through the formatter
|
||||
Then behave_parallel the formatter real stream output should not contain "Passing scenario"
|
||||
And behave_parallel the formatter real stream output should contain "Failing scenario"
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
Feature: CLI global options --data-dir, --config-path, and -v
|
||||
|
||||
Background:
|
||||
Given global options test env is clean
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --data-dir option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: --data-dir option is accepted with an existing directory
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: --data-dir option overrides CLEVERAGENTS_DATA_DIR for the invocation
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
|
||||
Scenario: --data-dir with a path that is an existing file fails with a clear error
|
||||
Given a temp file path is prepared
|
||||
When I run the global options CLI with data-dir as temp file and "version"
|
||||
Then the global options CLI should fail
|
||||
And the global options output should contain "--data-dir"
|
||||
|
||||
Scenario: --data-dir stores value in ctx.obj for subcommands
|
||||
Given a temp data dir is prepared
|
||||
When I run the global options CLI with data-dir flag and "version"
|
||||
Then the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --config-path option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: --config-path option is accepted with an existing file
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: --config-path overrides CLEVERAGENTS_CONFIG_PATH for the invocation
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
Scenario: --config-path with a non-existent file fails with a clear error
|
||||
When I run the global options CLI with invalid config-path and "version"
|
||||
Then the global options CLI should fail
|
||||
And the global options output should contain "--config-path"
|
||||
|
||||
Scenario: --config-path stores value in ctx.obj for subcommands
|
||||
Given a temp config file is prepared
|
||||
When I run the global options CLI with config-path flag and "version"
|
||||
Then the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Combining --data-dir and --config-path
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: --data-dir and --config-path can be combined
|
||||
Given a temp data dir is prepared
|
||||
And a temp config file is prepared
|
||||
When I run the global options CLI with both path flags and "version"
|
||||
Then the global options CLI should succeed
|
||||
And the CLEVERAGENTS_DATA_DIR env var should match the temp data dir
|
||||
And the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# -v verbosity option
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: No -v flag results in CRITICAL (silent) log level
|
||||
When I call main_callback with verbosity 0
|
||||
Then the global options log level should be "CRITICAL"
|
||||
|
||||
Scenario: Single -v flag results in ERROR log level
|
||||
When I call main_callback with verbosity 1
|
||||
Then the global options log level should be "ERROR"
|
||||
|
||||
Scenario: -vv results in WARNING log level
|
||||
When I call main_callback with verbosity 2
|
||||
Then the global options log level should be "WARNING"
|
||||
|
||||
Scenario: -vvv results in INFO log level
|
||||
When I call main_callback with verbosity 3
|
||||
Then the global options log level should be "INFO"
|
||||
|
||||
Scenario: -vvvv results in DEBUG log level
|
||||
When I call main_callback with verbosity 4
|
||||
Then the global options log level should be "DEBUG"
|
||||
|
||||
Scenario: -vvvvv or more results in DEBUG log level
|
||||
When I call main_callback with verbosity 5
|
||||
Then the global options log level should be "DEBUG"
|
||||
|
||||
@tdd_issue @tdd_issue_6785
|
||||
Scenario: -v flag is accepted by the CLI without crashing
|
||||
When I run the global options CLI with args "-v version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
Scenario: -vvv flags are accepted by the CLI without crashing
|
||||
When I run the global options CLI with args "-v -v -v version"
|
||||
Then the global options CLI should succeed
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# --help shows all three options
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: --help output includes --data-dir option
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "--data-dir"
|
||||
|
||||
Scenario: --help output includes --config-path option
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "--config-path"
|
||||
|
||||
Scenario: -v flag appears in --help output
|
||||
When I run the global options CLI with args "--help"
|
||||
Then the global options output should contain "-v"
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# -v verbosity stored in ctx.obj
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: verbose count is stored in ctx.obj
|
||||
When I call main_callback with verbosity 3
|
||||
Then the global options ctx obj "verbose" should be 3
|
||||
@@ -459,22 +459,6 @@ Feature: Consolidated Actor
|
||||
Then the config options should include the overrides
|
||||
|
||||
|
||||
Scenario: _extract_v2_actor parses v2 agents block
|
||||
When I extract v2 actor from a v2-style config with agents block
|
||||
Then the extracted provider and model should be set
|
||||
And the graph descriptor should contain agent info
|
||||
|
||||
|
||||
Scenario: _extract_v2_actor returns None for missing agents
|
||||
When I extract v2 actor from a config without agents block
|
||||
Then all extracted values should be None
|
||||
|
||||
|
||||
Scenario: _extract_v2_options extracts options from v2 config
|
||||
When I extract v2 options from a v2-style config
|
||||
Then the extracted options should contain expected keys
|
||||
|
||||
|
||||
Scenario: from_file loads and parses config from disk
|
||||
Given a temporary JSON config file with provider and model
|
||||
When I create an ActorConfiguration from the file
|
||||
@@ -876,6 +860,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/test-actor
|
||||
type: llm
|
||||
description: A test actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
@@ -888,6 +874,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text with schema_version "2.0":
|
||||
"""
|
||||
name: local/versioned
|
||||
type: llm
|
||||
description: A versioned actor
|
||||
provider: anthropic
|
||||
model: claude-3
|
||||
"""
|
||||
@@ -899,6 +887,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text with compiled_metadata:
|
||||
"""
|
||||
name: local/compiled
|
||||
type: llm
|
||||
description: A compiled actor
|
||||
provider: openai
|
||||
model: gpt-4o
|
||||
"""
|
||||
@@ -910,12 +900,16 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/updatable
|
||||
type: llm
|
||||
description: An updatable actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
And I update actor from YAML text:
|
||||
"""
|
||||
name: local/updatable
|
||||
type: llm
|
||||
description: An updatable actor
|
||||
provider: openai
|
||||
model: gpt-4-turbo
|
||||
"""
|
||||
@@ -927,6 +921,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/removable
|
||||
type: llm
|
||||
description: A removable actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
@@ -939,12 +935,16 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/alpha
|
||||
type: llm
|
||||
description: An alpha actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
And I add actor from YAML text with update:
|
||||
"""
|
||||
name: local/beta
|
||||
type: llm
|
||||
description: A beta actor
|
||||
provider: anthropic
|
||||
model: claude-3
|
||||
"""
|
||||
@@ -957,6 +957,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/auto-prefixed
|
||||
type: llm
|
||||
description: An auto-prefixed actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
@@ -968,6 +970,8 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/fetchable
|
||||
type: llm
|
||||
description: A fetchable actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
@@ -978,6 +982,8 @@ Feature: Consolidated Actor
|
||||
Given an actor registry with no configured providers for persistence
|
||||
When I attempt to add actor from YAML text without name:
|
||||
"""
|
||||
type: llm
|
||||
description: No name actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
@@ -989,12 +995,16 @@ Feature: Consolidated Actor
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/duplicate
|
||||
type: llm
|
||||
description: A duplicate actor
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
And I attempt to add duplicate actor from YAML text:
|
||||
"""
|
||||
name: local/duplicate
|
||||
type: llm
|
||||
description: A duplicate actor
|
||||
provider: openai
|
||||
model: gpt-4-turbo
|
||||
"""
|
||||
@@ -1008,17 +1018,7 @@ Feature: Consolidated Actor
|
||||
And the actor "local/legacy-yaml" should have schema_version "1.5"
|
||||
|
||||
|
||||
Scenario: Default schema version is applied when not specified
|
||||
Given an actor registry with no configured providers for persistence
|
||||
When I add actor from YAML text:
|
||||
"""
|
||||
name: local/default-version
|
||||
provider: openai
|
||||
model: gpt-4
|
||||
"""
|
||||
Then the actor "local/default-version" should have schema_version "1.0"
|
||||
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Originally from: actor_runtime.feature
|
||||
# Feature: Tool-Calling Actor Runtime
|
||||
|
||||
@@ -1159,7 +1159,7 @@ Feature: Consolidated Domain Models
|
||||
|
||||
Scenario: CLI dict has required fields
|
||||
Given a session with some messages
|
||||
When I get the session CLI dict
|
||||
When I get the session model CLI dict
|
||||
Then the session cli dict should have key "session_summary"
|
||||
And the session cli dict should have key "token_usage"
|
||||
And the session cli dict session_summary should have key "id"
|
||||
@@ -1170,34 +1170,34 @@ Feature: Consolidated Domain Models
|
||||
|
||||
Scenario: CLI dict includes recent messages
|
||||
Given a session with some messages
|
||||
When I get the session CLI dict
|
||||
When I get the session model CLI dict
|
||||
Then the session cli dict should have key "recent_messages"
|
||||
And the session cli dict recent_messages text key should be "text"
|
||||
|
||||
|
||||
Scenario: CLI dict includes actor when set
|
||||
When I create a session with actor name "local/orchestrator"
|
||||
And I get the session CLI dict
|
||||
And I get the session model CLI dict
|
||||
Then the session cli dict session_summary should have key "actor"
|
||||
|
||||
|
||||
Scenario: CLI dict includes automation when set
|
||||
When I create a session with automation "review"
|
||||
And I get the session CLI dict
|
||||
And I get the session model CLI dict
|
||||
Then the session cli dict session_summary should have key "automation"
|
||||
And the session cli dict session_summary automation should be "review"
|
||||
|
||||
|
||||
Scenario: CLI dict linked_plans uses spec-compliant objects
|
||||
Given a session with linked plans
|
||||
When I get the session CLI dict
|
||||
When I get the session model CLI dict
|
||||
Then the session cli dict should have key "linked_plans"
|
||||
And the session cli dict linked_plans should contain plan_id field
|
||||
|
||||
|
||||
Scenario: CLI dict token_usage estimated_cost is formatted string
|
||||
Given a session with token usage cost 0.0184
|
||||
When I get the session CLI dict
|
||||
When I get the session model CLI dict
|
||||
Then the session cli dict token_usage estimated_cost should be a string starting with "$"
|
||||
|
||||
# ---- Empty Session Properties ----
|
||||
|
||||
@@ -140,7 +140,7 @@ Feature: Consolidated Misc
|
||||
And the operations list should contain "registry.list_tools"
|
||||
And the operations list should contain "context.get"
|
||||
And the operations list should contain "event.subscribe"
|
||||
And the operations list should have 42 items
|
||||
And the operations list should have 44 items
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# A2aHttpTransport — all stubs raise A2aNotAvailableError
|
||||
@@ -639,7 +639,7 @@ Feature: Consolidated Misc
|
||||
Then the m6 smoke operations should include "session.create"
|
||||
And the m6 smoke operations should include "plan.execute"
|
||||
And the m6 smoke operations should include "event.subscribe"
|
||||
And the m6 smoke operations count should be 42
|
||||
And the m6 smoke operations count should be 44
|
||||
|
||||
# --- A2A event queue ---
|
||||
|
||||
|
||||
@@ -352,24 +352,24 @@ Feature: Consolidated Plan Model Lifecycle
|
||||
|
||||
|
||||
Scenario: New plan starts in STRATEGIZE with QUEUED processing state
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan phase should be "strategize"
|
||||
And the plan processing state should be "queued"
|
||||
|
||||
|
||||
Scenario: Strategize phase uses ProcessingState
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan processing state should be "queued"
|
||||
|
||||
|
||||
Scenario: Plan in strategize phase defaults to QUEUED processing state
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan processing state should be "queued"
|
||||
And the plan state should be "queued"
|
||||
|
||||
|
||||
Scenario: Strategize phase defaults processing_state to QUEUED
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan processing state should be "queued"
|
||||
|
||||
# Plan Identity Tests
|
||||
@@ -401,12 +401,12 @@ Feature: Consolidated Plan Model Lifecycle
|
||||
|
||||
|
||||
Scenario: Plan in STRATEGIZE with QUEUED cannot transition
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan cannot transition to next phase
|
||||
|
||||
|
||||
Scenario: New plan in STRATEGIZE with QUEUED cannot transition
|
||||
Given I create a plan in strategize phase
|
||||
Given I create a PlanModel in strategize phase
|
||||
Then the plan cannot transition to next phase
|
||||
|
||||
|
||||
@@ -455,17 +455,17 @@ Feature: Consolidated Plan Model Lifecycle
|
||||
|
||||
|
||||
Scenario: Plan in errored state is_errored returns True
|
||||
Given I create a plan in strategize phase with errored state
|
||||
Given I create a PlanModel in strategize phase with errored state
|
||||
Then the plan should be errored
|
||||
|
||||
|
||||
Scenario: Plan in processing state is_errored returns False
|
||||
Given I create a plan in strategize phase with processing state
|
||||
Given I create a PlanModel in strategize phase with processing state
|
||||
Then the plan should not be errored
|
||||
|
||||
|
||||
Scenario: Cancelled plan is terminal
|
||||
Given I create a plan in strategize phase with cancelled state
|
||||
Given I create a PlanModel in strategize phase with cancelled state
|
||||
Then the plan should be terminal
|
||||
|
||||
# Model Validation Tests
|
||||
|
||||
@@ -401,3 +401,32 @@ Feature: Decision recording and snapshot store
|
||||
And I record a strategy_choice decision for plan "P1" with question "After restart"
|
||||
Then the dsvc decision sequence number should be 2
|
||||
And the dsvc next sequence for plan "P1" should be 3
|
||||
|
||||
# --- Full context snapshot (issue #9056) ---
|
||||
|
||||
Scenario: Strategize phase records decisions with full context snapshots
|
||||
Given a plan lifecycle service with decision service wired
|
||||
And an action "local/test-action" for strategize snapshot test
|
||||
And a plan created from "local/test-action" with project "proj-snapshot"
|
||||
When I start strategize for the snapshot test plan
|
||||
Then the strategize decision should have a non-empty hot_context_hash
|
||||
And the strategize decision should have a non-empty hot_context_ref
|
||||
And the strategize decision hot_context_ref should start with "plan:"
|
||||
And the strategize decision should have a non-empty actor_state_ref
|
||||
And the strategize decision should have relevant_resources populated
|
||||
|
||||
Scenario: Strategize context snapshot hash is content-addressable
|
||||
Given a plan lifecycle service with decision service wired
|
||||
And an action "local/test-action-hash" for strategize snapshot test
|
||||
And a plan created from "local/test-action-hash" with project "proj-hash"
|
||||
When I start strategize for the snapshot test plan
|
||||
Then the strategize decision hot_context_hash should start with "sha256:"
|
||||
And the strategize decision hot_context_hash should be 71 characters long
|
||||
|
||||
Scenario: Strategize context snapshot without projects has empty relevant_resources
|
||||
Given a plan lifecycle service with decision service wired
|
||||
And an action "local/no-project-action" for strategize snapshot test
|
||||
And a plan created from "local/no-project-action" without projects
|
||||
When I start strategize for the snapshot test plan
|
||||
Then the strategize decision should have a non-empty hot_context_hash
|
||||
And the strategize decision should have empty relevant_resources
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
Feature: Devcontainer auto-discovery wired into git-checkout and fs-directory handlers
|
||||
As a CleverAgents user
|
||||
I want devcontainer configurations to be automatically discovered
|
||||
When I register a git-checkout or fs-directory resource
|
||||
So that devcontainer-instance child resources are created without manual intervention
|
||||
|
||||
Scenario: git-checkout discover_children finds root devcontainer config
|
||||
Given dcwire a git repo with a ".devcontainer/devcontainer.json" file
|
||||
When dcwire I call discover_children on the git-checkout resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
And dcwire the devcontainer child has provisioning_state "discovered"
|
||||
And dcwire the devcontainer child has devcontainer_json_path set
|
||||
|
||||
Scenario: git-checkout discover_children finds named devcontainer config
|
||||
Given dcwire a git repo with a ".devcontainer/api/devcontainer.json" named config
|
||||
When dcwire I call discover_children on the git-checkout resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-api"
|
||||
And dcwire the devcontainer child has config_name "api"
|
||||
|
||||
Scenario: git-checkout discover_children with no devcontainer returns only directories
|
||||
Given dcwire a git repo with no devcontainer configuration
|
||||
When dcwire I call discover_children on the git-checkout resource
|
||||
Then dcwire no devcontainer-instance children are present
|
||||
|
||||
Scenario: git-checkout discover_children includes both fs-directory and devcontainer children
|
||||
Given dcwire a git repo with a subdirectory "src" and a ".devcontainer/devcontainer.json" file
|
||||
When dcwire I call discover_children on the git-checkout resource
|
||||
Then dcwire the children include a "fs-directory" resource named "src"
|
||||
And dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
|
||||
Scenario: fs-directory discover_children finds root devcontainer config
|
||||
Given dcwire a filesystem directory with a ".devcontainer/devcontainer.json" file
|
||||
When dcwire I call discover_children on the fs-directory resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
And dcwire the devcontainer child has provisioning_state "discovered"
|
||||
|
||||
Scenario: fs-directory discover_children finds named devcontainer config
|
||||
Given dcwire a filesystem directory with a ".devcontainer/frontend/devcontainer.json" named config
|
||||
When dcwire I call discover_children on the fs-directory resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-frontend"
|
||||
And dcwire the devcontainer child has config_name "frontend"
|
||||
|
||||
Scenario: fs-directory discover_children with no devcontainer returns only directories
|
||||
Given dcwire a filesystem directory with no devcontainer configuration
|
||||
When dcwire I call discover_children on the fs-directory resource
|
||||
Then dcwire no devcontainer-instance children are present
|
||||
|
||||
Scenario: fs-directory discover_children includes both fs-directory and devcontainer children
|
||||
Given dcwire a filesystem directory with a subdirectory "lib" and a ".devcontainer/devcontainer.json" file
|
||||
When dcwire I call discover_children on the fs-directory resource
|
||||
Then dcwire the children include a "fs-directory" resource named "lib"
|
||||
And dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
|
||||
Scenario: git-checkout discover_children finds root-level .devcontainer.json
|
||||
Given dcwire a git repo with a root-level ".devcontainer.json" file
|
||||
When dcwire I call discover_children on the git-checkout resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
|
||||
Scenario: fs-directory discover_children finds root-level .devcontainer.json
|
||||
Given dcwire a filesystem directory with a root-level ".devcontainer.json" file
|
||||
When dcwire I call discover_children on the fs-directory resource
|
||||
Then dcwire the children include a "devcontainer-instance" resource named "devcontainer-default"
|
||||
@@ -180,19 +180,19 @@ Feature: Plan edge case scenarios
|
||||
|
||||
Scenario: Plan model rejects empty description
|
||||
When I try to create an edge case plan with empty description
|
||||
Then a Pydantic validation error should be raised
|
||||
Then an Edge Case Pydantic validation error should be raised
|
||||
|
||||
Scenario: Plan model rejects invalid phase value
|
||||
When I try to create an edge case plan with invalid phase value
|
||||
Then a Pydantic validation error should be raised
|
||||
Then an Edge Case Pydantic validation error should be raised
|
||||
|
||||
Scenario: NamespacedName rejects invalid characters in namespace
|
||||
When I try to parse a namespaced name with special characters "inv@lid/action"
|
||||
Then a Pydantic validation error should be raised
|
||||
Then an Edge Case Pydantic validation error should be raised
|
||||
|
||||
Scenario: NamespacedName rejects invalid characters in name
|
||||
When I try to parse a namespaced name with special chars in name "local/my action!"
|
||||
Then a Pydantic validation error should be raised
|
||||
Then an Edge Case Pydantic validation error should be raised
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# Section 4: Rollback edge cases
|
||||
|
||||
+21
-8
@@ -1,6 +1,7 @@
|
||||
"""Behave environment setup for feature tests."""
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -335,13 +336,13 @@ def before_all(context):
|
||||
# Use per-process unique database paths so parallel test subprocesses
|
||||
# (behave-parallel) never contend on the same SQLite file.
|
||||
if "CLEVERAGENTS_DATABASE_URL" not in os.environ:
|
||||
os.environ["CLEVERAGENTS_DATABASE_URL"] = (
|
||||
f"sqlite:///{tempfile.mktemp(suffix='.db', prefix='cleveragents_')}"
|
||||
)
|
||||
_fd, _db_path = tempfile.mkstemp(suffix=".db", prefix="cleveragents_")
|
||||
os.close(_fd)
|
||||
os.environ["CLEVERAGENTS_DATABASE_URL"] = f"sqlite:///{_db_path}"
|
||||
if "CLEVERAGENTS_TEST_DATABASE_URL" not in os.environ:
|
||||
os.environ["CLEVERAGENTS_TEST_DATABASE_URL"] = (
|
||||
f"sqlite:///{tempfile.mktemp(suffix='.db', prefix='cleveragents_test_')}"
|
||||
)
|
||||
_fd, _db_path = tempfile.mkstemp(suffix=".db", prefix="cleveragents_test_")
|
||||
os.close(_fd)
|
||||
os.environ["CLEVERAGENTS_TEST_DATABASE_URL"] = f"sqlite:///{_db_path}"
|
||||
os.environ.setdefault("BEHAVE_TESTING", "true")
|
||||
|
||||
# Set up mock AI provider for all tests
|
||||
@@ -453,8 +454,15 @@ def _ensure_template_db() -> None:
|
||||
os.environ["CLEVERAGENTS_TEMPLATE_DB"] = str(template_path.resolve())
|
||||
return
|
||||
|
||||
lock_path = template_path.with_suffix(".db.lock")
|
||||
_lock_fd = -1
|
||||
try:
|
||||
# Import the template creation script
|
||||
_lock_fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR)
|
||||
fcntl.flock(_lock_fd, fcntl.LOCK_EX)
|
||||
if template_path.is_file():
|
||||
os.environ["CLEVERAGENTS_TEMPLATE_DB"] = str(template_path.resolve())
|
||||
return
|
||||
|
||||
scripts_dir = Path(__file__).parent.parent / "scripts"
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
from create_template_db import create_template
|
||||
@@ -463,6 +471,10 @@ def _ensure_template_db() -> None:
|
||||
os.environ["CLEVERAGENTS_TEMPLATE_DB"] = str(template_path.resolve())
|
||||
except Exception:
|
||||
pass # Fall back to normal Alembic migrations
|
||||
finally:
|
||||
if _lock_fd >= 0:
|
||||
fcntl.flock(_lock_fd, fcntl.LOCK_UN)
|
||||
os.close(_lock_fd)
|
||||
|
||||
|
||||
def _install_template_db_patch() -> None:
|
||||
@@ -636,7 +648,8 @@ def before_scenario(context, scenario):
|
||||
("CLEVERAGENTS_DATABASE_URL", "cleveragents_"),
|
||||
("CLEVERAGENTS_TEST_DATABASE_URL", "cleveragents_test_"),
|
||||
):
|
||||
db_path = tempfile.mktemp(suffix=".db", prefix=prefix)
|
||||
_fd, db_path = tempfile.mkstemp(suffix=".db", prefix=prefix)
|
||||
os.close(_fd)
|
||||
os.environ[env_var] = f"sqlite:///{db_path}"
|
||||
context._scenario_db_paths.append(db_path)
|
||||
|
||||
|
||||
@@ -46,6 +46,31 @@ Feature: Execute-phase context assembler coverage
|
||||
When epcov I check path matching for "src/foo.py" with exclude "src/secret*"
|
||||
Then epcov the path should match
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: epcov path matches absolute path against relative include glob
|
||||
When epcov I check path matching for "/app/.opencode/skills/SKILL.md" with include ".opencode/**"
|
||||
Then epcov the path should match
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: epcov path matches absolute path against relative exclude glob
|
||||
When epcov I check path matching for "/app/.opencode/skills/SKILL.md" with exclude ".opencode/**"
|
||||
Then epcov the path should not match
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: epcov path matches absolute path against relative include glob with wildcard
|
||||
When epcov I check path matching for "/app/docs/readme.md" with include "docs/*"
|
||||
Then epcov the path should match
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: epcov path matches absolute path not matching relative include glob
|
||||
When epcov I check path matching for "/app/src/main.py" with include "docs/*"
|
||||
Then epcov the path should not match
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: epcov relative path is excluded by trailing ** glob
|
||||
When epcov I check path matching for "build/debug/output.log" with exclude "build/**"
|
||||
Then epcov the path should not match
|
||||
|
||||
# ---- _resource_matches static method ----
|
||||
|
||||
Scenario: epcov resource matches with no rules passes all
|
||||
|
||||
@@ -109,7 +109,7 @@ Feature: M6 autonomy acceptance smoke tests
|
||||
Then the m6 smoke operations should include "session.create"
|
||||
And the m6 smoke operations should include "plan.execute"
|
||||
And the m6 smoke operations should include "event.subscribe"
|
||||
And the m6 smoke operations count should be 42
|
||||
And the m6 smoke operations count should be 44
|
||||
|
||||
# --- A2A event queue (AC-2: event queue publish/subscribe) ---
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ Feature: PlanExecutor Coverage Boost
|
||||
Scenario: _try_rollback_to_last_checkpoint returns False when no sandbox is resolvable
|
||||
Given a PlanExecutor with a checkpoint manager but no sandbox source
|
||||
When I attempt to rollback to the last checkpoint
|
||||
Then the rollback result should be False
|
||||
Then the executor rollback result should be False
|
||||
|
||||
# --- _resolve_sandbox_for_checkpoint via execution_context (line 458) ---
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ Feature: Plan explain and decision tree CLI commands
|
||||
Given a test decision for explain
|
||||
When I format the explain dict as json
|
||||
Then the json output should contain "decision_id"
|
||||
And the json output should be valid json
|
||||
And the plan explain json output should be valid
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# plan explain - yaml format
|
||||
@@ -107,12 +107,12 @@ Feature: Plan explain and decision tree CLI commands
|
||||
# plan tree - json format
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@tdd_issue @tdd_issue_4254 @tdd_expected_fail
|
||||
@tdd_issue @tdd_issue_4254
|
||||
Scenario: Tree with json format
|
||||
Given a set of test decisions forming a tree
|
||||
When I format the tree as json
|
||||
Then the json tree output should be valid json
|
||||
And the json tree output should contain "decision_id"
|
||||
Then the json tree output should be a valid json envelope
|
||||
And the json tree output should contain "command"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# plan tree - yaml format
|
||||
|
||||
@@ -65,13 +65,13 @@ Feature: Plan explain and tree CLI command coverage
|
||||
Given pec a mock DecisionService returning a list of decisions
|
||||
When pec I invoke "tree" with format "json"
|
||||
Then pec the exit code should be 0
|
||||
And pec the output should be valid json list
|
||||
And pec the output should be valid json envelope
|
||||
|
||||
Scenario: Tree CLI renders yaml format
|
||||
Given pec a mock DecisionService returning a list of decisions
|
||||
When pec I invoke "tree" with format "yaml"
|
||||
Then pec the exit code should be 0
|
||||
And pec the output should contain "decision_id:"
|
||||
And pec the output should contain "command:"
|
||||
|
||||
Scenario: Tree CLI renders table format
|
||||
Given pec a mock DecisionService returning a list of decisions
|
||||
|
||||
@@ -69,9 +69,20 @@ Feature: Plan Generation LangGraph Coverage
|
||||
|
||||
Scenario: Should retry returns end when max retries reached
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 4
|
||||
Then the langgraph retry decision should be "end"
|
||||
|
||||
Scenario: Should retry still retries when retry_count equals max_retries
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
When I check langgraph should_retry with FAIL validation and retry_count 3
|
||||
Then the langgraph retry decision should be "retry"
|
||||
|
||||
Scenario: Validate node increments retry_count in returned state
|
||||
Given I have a langgraph PlanGenerationGraph instance
|
||||
When I execute the langgraph validate node with no changes
|
||||
Then the langgraph validation status should be "FAIL"
|
||||
And the langgraph validate result should include retry_count incremented from 0 to 1
|
||||
|
||||
Scenario: Validate node fails when no changes provided
|
||||
Given I have a langgraph PlanGenerationGraph instance
|
||||
When I execute the langgraph validate node with no changes
|
||||
|
||||
@@ -78,12 +78,13 @@ Feature: Plan Generation Graph Uncovered Lines Coverage
|
||||
And the langgraph async result should have generated_changes
|
||||
And the langgraph async result should have validation_result
|
||||
|
||||
# Testing line 403: Retry count increment in should_retry
|
||||
Scenario: Should retry increments retry count correctly
|
||||
# _should_retry is a read-only conditional-edge function; retry_count is
|
||||
# incremented inside _validate (a node) so LangGraph persists the new value.
|
||||
Scenario: Should retry does not mutate state and returns retry when retries remain
|
||||
Given I have a langgraph PlanGenerationGraph instance with max_retries 3
|
||||
And I have a langgraph state with retry_count 1
|
||||
When I check uncovered langgraph should_retry with FAIL validation and retry_count 1
|
||||
Then the uncovered langgraph state retry_count should be incremented to 2
|
||||
Then the uncovered langgraph state retry_count should not be mutated and remain 1
|
||||
And the langgraph retry decision should be "retry"
|
||||
|
||||
# Testing full workflow with retry logic
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
Feature: Plugin Loader Coverage Boost
|
||||
Scenarios targeting uncovered lines in the PluginLoader class:
|
||||
- Lines 203-209: entry point load failure exception handler
|
||||
- Lines 242, 244-246: validate_protocol fallback to issubclass when instantiation fails
|
||||
- Lines 247-248: validate_protocol issubclass raises TypeError
|
||||
- validate_protocol: issubclass succeeds (class satisfies protocol structurally)
|
||||
- validate_protocol: issubclass raises TypeError with unverifiable protocol
|
||||
- validate_protocol: issubclass returns False (class missing required members)
|
||||
|
||||
Background:
|
||||
Given the plugin loader module is imported
|
||||
@@ -18,17 +19,17 @@ Feature: Plugin Loader Coverage Boost
|
||||
And the failed entry point should have been logged as a warning
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass succeeds (lines 242, 244-246)
|
||||
# validate_protocol: issubclass succeeds for class satisfying protocol
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: validate_protocol falls back to issubclass when instantiation fails
|
||||
Scenario: validate_protocol returns True when class satisfies protocol via issubclass
|
||||
Given I have a class that requires constructor arguments
|
||||
And I have a runtime checkable protocol the class satisfies via issubclass
|
||||
When I call validate_protocol with the non-instantiable class and protocol
|
||||
Then validate_protocol should return True
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass raises TypeError (lines 247-248)
|
||||
# validate_protocol: issubclass raises TypeError for unverifiable protocol
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: validate_protocol raises ProtocolMismatchError when issubclass raises TypeError
|
||||
@@ -38,7 +39,7 @@ Feature: Plugin Loader Coverage Boost
|
||||
Then a plugin-loader ProtocolMismatchError should be raised
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass returns False
|
||||
# validate_protocol: issubclass returns False (class missing required members)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Scenario: validate_protocol raises ProtocolMismatchError when issubclass returns False
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
@mock_only
|
||||
Feature: PR Compliance Checklist in Implementation Pool Supervisor
|
||||
|
||||
As a pool supervisor
|
||||
I want to pass a mandatory PR compliance checklist to every worker prompt
|
||||
So that implementation workers complete all required items before creating a PR and avoid systemic merge blockers
|
||||
|
||||
Background:
|
||||
Given the implementation-pool-supervisor.md agent definition exists
|
||||
|
||||
Scenario: Pool supervisor worker prompt includes the PR compliance checklist
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes the PR compliance checklist section
|
||||
And Pool: the checklist is marked as MANDATORY
|
||||
|
||||
Scenario: Checklist item 1 — CHANGELOG.md update required
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a CHANGELOG.md checklist item
|
||||
And Pool: the item instructs workers to add an entry under the Unreleased section
|
||||
|
||||
Scenario: Checklist item 2 — CONTRIBUTORS.md update required
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a CONTRIBUTORS.md checklist item
|
||||
And Pool: the item instructs workers to add or update their contribution entry
|
||||
|
||||
Scenario: Checklist item 3 — commit footer required
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a commit footer checklist item
|
||||
And Pool: the item specifies the ISSUES CLOSED footer format
|
||||
|
||||
Scenario: Checklist item 4 — CI must pass before PR creation
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a CI passes checklist item
|
||||
And Pool: the item instructs workers to verify all quality gates are green
|
||||
|
||||
Scenario: Checklist item 5 — BDD/Behave tests required
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a BDD tests checklist item
|
||||
And Pool: the item instructs workers to add or update Behave feature files
|
||||
|
||||
Scenario: Checklist item 6 — Epic reference required in PR description
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes an Epic reference checklist item
|
||||
And Pool: the item instructs workers to reference the parent Epic issue number
|
||||
|
||||
Scenario: Checklist item 7 — Labels must be applied
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a labels checklist item
|
||||
And Pool: the item instructs workers to apply labels via forgejo-label-manager
|
||||
|
||||
Scenario: Checklist item 8 — Milestone must be assigned
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body includes a milestone checklist item
|
||||
And Pool: the item instructs workers to assign the earliest open milestone
|
||||
|
||||
Scenario: All 8 checklist items are present in the worker prompt
|
||||
When I read the pool supervisor agent definition
|
||||
Then Pool: worker prompt body contains all 8 mandatory checklist items
|
||||
@@ -29,3 +29,10 @@ Feature: Project context phase analysis summaries
|
||||
When I compute project context phase analysis with budget 1000
|
||||
Then execute phase should have fewer tokens than strategize phase
|
||||
And apply phase should have fewer or equal tokens than execute phase
|
||||
|
||||
@tdd_issue @tdd_issue_10972
|
||||
Scenario: Absolute path fragments are correctly excluded by relative exclude globs
|
||||
Given a phase analysis policy with opencode exclude paths
|
||||
And an absolute path fragment for phase analysis
|
||||
When I compute project context phase analysis with budget 2000
|
||||
Then strategize phase should exclude the absolute path fragment
|
||||
|
||||
@@ -125,7 +125,7 @@ Feature: Namespaced project repository operations
|
||||
|
||||
Scenario: Remove a non-existent link returns False
|
||||
When I remove a link with id "00000000000000000000000099"
|
||||
Then the remove result should be False
|
||||
Then the project repo remove result should be False
|
||||
|
||||
Scenario: Create link with read_only flag
|
||||
Given a namespaced project "local/ro-proj" exists in the repository
|
||||
|
||||
@@ -76,7 +76,7 @@ Feature: Retry Policy Wiring for Services
|
||||
@registry
|
||||
Scenario: Per-service policy defaults with config overrides
|
||||
Given I have Settings with retry_service_overrides setting plan_service max_attempts to 10
|
||||
When I create a ServiceRetryWiring from those Settings
|
||||
When I create a ServiceRetryWiring from those retry Settings
|
||||
Then the plan_service policy should have max_attempts 10
|
||||
|
||||
@registry
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
@tdd_issue @tdd_issue_9055
|
||||
Feature: PyYAML is a declared project dependency with minimum secure version
|
||||
As a CleverAgents developer
|
||||
I want the PyYAML dependency to be listed in pyproject.toml with a minimum secure version
|
||||
So that vulnerable older versions of PyYAML cannot be installed
|
||||
|
||||
Background:
|
||||
Given the pyproject.toml file exists at "pyproject.toml"
|
||||
|
||||
@tdd_issue @tdd_issue_9055
|
||||
Scenario: PyYAML is listed in project dependencies with minimum version constraint
|
||||
When I read the project dependencies from pyproject.toml
|
||||
Then the dependency list should include a package matching "pyyaml>="
|
||||
|
||||
@tdd_issue @tdd_issue_9055
|
||||
Scenario: PyYAML minimum version is >=6.0.3
|
||||
When I read the PyYAML dependency specification from pyproject.toml
|
||||
Then the minimum version should be at least "6.0.3"
|
||||
|
||||
@tdd_issue @tdd_issue_9055
|
||||
Scenario: yaml module is importable as a project dependency
|
||||
When I attempt to import the "yaml" module
|
||||
Then the import should succeed without errors
|
||||
@@ -0,0 +1,58 @@
|
||||
Feature: Real LLM actor invocation in session tell
|
||||
As a user of CleverAgents
|
||||
I want `agents session tell` to invoke the real orchestrator actor
|
||||
So that I get meaningful AI responses instead of stub acknowledgements
|
||||
|
||||
Background:
|
||||
Given a session tell LLM mock environment
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: Real LLM response is returned and persisted
|
||||
Given a session with actor "openai/gpt-4" exists
|
||||
When I invoke session tell with prompt "What can you do?"
|
||||
Then the tell command returns the LLM response
|
||||
And the assistant message is persisted to the session
|
||||
And token usage is recorded
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: --stream flag yields incremental tokens
|
||||
Given a session with actor "openai/gpt-4" exists
|
||||
When I invoke session tell with --stream and prompt "Hello"
|
||||
Then the streamed output contains the LLM response tokens
|
||||
And the assistant message is persisted after streaming
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: No actor configured raises clear error with exit code 1
|
||||
Given a session with no actor exists
|
||||
When I invoke session tell with prompt "Hello"
|
||||
Then the tell command exits with code 1
|
||||
And the error output mentions actor configuration
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: --actor flag overrides the session's bound actor
|
||||
Given a session with actor "anthropic/claude-3-haiku" exists
|
||||
When I invoke session tell with --actor "openai/gpt-4" and prompt "Hello"
|
||||
Then the tell command uses the override actor "openai/gpt-4"
|
||||
And the tell command returns the LLM response
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: --format json output includes usage object
|
||||
Given a session with actor "openai/gpt-4" exists
|
||||
When I invoke session tell with --format json and prompt "What can you do?"
|
||||
Then the tell output is valid JSON with a data envelope
|
||||
And the data section contains the session_id
|
||||
And the data section contains a usage object with expected keys
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: --stream --format json produces valid JSON with usage
|
||||
Given a session with actor "openai/gpt-4" exists
|
||||
When I invoke session tell with --stream --format json and prompt "Hello"
|
||||
Then the tell output is valid JSON with a data envelope
|
||||
And the data section contains a usage object with expected keys
|
||||
|
||||
@session_tell_llm
|
||||
Scenario: Usage panel appears in Rich output
|
||||
Given a session with actor "openai/gpt-4" exists
|
||||
When I invoke session tell with prompt "What can you do?"
|
||||
Then the tell command returns the LLM response
|
||||
And the output contains a Usage panel
|
||||
@@ -0,0 +1,105 @@
|
||||
Feature: Session workflow coverage boost
|
||||
Coverage boost for session_workflow.py helper functions (M1, n4).
|
||||
|
||||
Background:
|
||||
Given the session workflow coverage environment is set up
|
||||
|
||||
# _extract_content
|
||||
Scenario: _extract_content extracts text from content attribute
|
||||
Given coverage boost a mock response with content "hello world"
|
||||
When coverage boost _extract_content is called
|
||||
Then coverage boost the extracted result should be "hello world"
|
||||
|
||||
Scenario: _extract_content falls back to text attribute
|
||||
Given coverage boost a mock response with text "from text attr" and no content
|
||||
When coverage boost _extract_content is called
|
||||
Then coverage boost the extracted result should be "from text attr"
|
||||
|
||||
Scenario: _extract_content handles list content
|
||||
Given coverage boost a mock response with list content containing text dicts and plain strings
|
||||
When coverage boost _extract_content is called
|
||||
Then coverage boost the result should contain the concatenated texts
|
||||
|
||||
Scenario: _extract_content falls back to str for unknown types
|
||||
Given coverage boost a mock response with no content or text attribute
|
||||
When coverage boost _extract_content is called
|
||||
Then coverage boost the result should be a string
|
||||
|
||||
# _extract_token_usage
|
||||
Scenario: _extract_token_usage reads from response_metadata.usage
|
||||
Given coverage boost a mock response with response_metadata usage input_tokens=100 output_tokens=50
|
||||
When coverage boost _extract_token_usage is called
|
||||
Then coverage boost input tokens should be 100 and output tokens should be 50
|
||||
|
||||
Scenario: _extract_token_usage reads from response_metadata.token_usage
|
||||
Given coverage boost a mock response with response_metadata token_usage input_tokens=200 output_tokens=100
|
||||
When coverage boost _extract_token_usage is called
|
||||
Then coverage boost input tokens should be 200 and output tokens should be 100
|
||||
|
||||
Scenario: _extract_token_usage reads prompt_tokens and completion_tokens
|
||||
Given coverage boost a mock response with response_metadata usage prompt_tokens=300 completion_tokens=150
|
||||
When coverage boost _extract_token_usage is called
|
||||
Then coverage boost input tokens should be 300 and output tokens should be 150
|
||||
|
||||
Scenario: _extract_token_usage reads from usage_metadata
|
||||
Given coverage boost a mock response with usage_metadata input_tokens=400 output_tokens=200
|
||||
When coverage boost _extract_token_usage is called
|
||||
Then coverage boost input tokens should be 400 and output tokens should be 200
|
||||
|
||||
Scenario: _extract_token_usage returns zeros for no metadata
|
||||
Given coverage boost a mock response with no usage metadata
|
||||
When coverage boost _extract_token_usage is called
|
||||
Then coverage boost input tokens should be 0 and output tokens should be 0
|
||||
|
||||
# _estimate_cost
|
||||
Scenario: _estimate_cost computes cost from token counts
|
||||
Given coverage boost input tokens 1000 and output tokens 500
|
||||
When coverage boost _estimate_cost is called
|
||||
Then coverage boost the estimated cost should be positive
|
||||
|
||||
# _history_to_langchain_messages
|
||||
Scenario: _history_to_langchain_messages converts all roles
|
||||
Given coverage boost session messages with roles SYSTEM, USER, ASSISTANT, and TOOL
|
||||
When coverage boost _history_to_langchain_messages is called
|
||||
Then coverage boost the result should contain SystemMessage, HumanMessage, AIMessage, and ToolMessage
|
||||
|
||||
Scenario: _history_to_langchain_messages treats unknown role as human
|
||||
Given coverage boost a session message with an unknown role
|
||||
When coverage boost _history_to_langchain_messages is called
|
||||
Then coverage boost the result should contain a HumanMessage
|
||||
|
||||
Scenario: _history_to_langchain_messages handles empty list
|
||||
Given coverage boost an empty list of session messages
|
||||
When coverage boost _history_to_langchain_messages is called
|
||||
Then coverage boost the result should be an empty list
|
||||
|
||||
# LangChainSessionCaller.invoke() tool_results branch
|
||||
Scenario: LangChainSessionCaller.invoke appends tool results
|
||||
Given coverage boost a LangChainSessionCaller with a stub LLM and empty history
|
||||
When coverage boost invoke is called with tool_results containing one success and one failure
|
||||
Then coverage boost the accumulated messages should include tool result messages
|
||||
|
||||
# LangChainSessionCaller.invoke() with tool_calls in response
|
||||
Scenario: LangChainSessionCaller.invoke extracts tool calls from response
|
||||
Given coverage boost a LangChainSessionCaller with a stub LLM that returns tool calls
|
||||
When coverage boost invoke is called for the first time
|
||||
Then coverage boost the LLMResponse should contain the extracted tool calls
|
||||
|
||||
# _build_lc_messages_from_history
|
||||
Scenario: _build_lc_messages_from_history adds system prompt when absent
|
||||
Given coverage boost a SessionWorkflow with a stub service and no registry
|
||||
And coverage boost session history without a system message
|
||||
When coverage boost _build_lc_messages_from_history is called with a prompt
|
||||
Then coverage boost the first message should be a SystemMessage with the session system prompt
|
||||
|
||||
# _MinimalStubLLM
|
||||
Scenario: _MinimalStubLLM.invoke returns stub response
|
||||
Given coverage boost a _MinimalStubLLM instance
|
||||
When coverage boost invoke on the stub is called
|
||||
Then coverage boost the stub response content should be "(no LLM configured)"
|
||||
And coverage boost the stub response should have empty tool_calls
|
||||
|
||||
Scenario: _MinimalStubLLM.stream yields stub chunk
|
||||
Given coverage boost a _MinimalStubLLM instance
|
||||
When coverage boost stream on the stub is called
|
||||
Then coverage boost it should yield a chunk with content "(no LLM configured)"
|
||||
@@ -127,7 +127,8 @@ def step_call_notify_facade(context: Any, operation: str) -> None:
|
||||
@then("the facade should support all 42 operations")
|
||||
def step_check_42_operations(context: Any) -> None:
|
||||
ops = context.facade.list_operations()
|
||||
assert len(ops) == 42, f"Expected 42 operations, got {len(ops)}: {ops}"
|
||||
# 42 original ops + 2 standard A2A ops (message/send, message/stream) = 44
|
||||
assert len(ops) == 44, f"Expected 44 operations, got {len(ops)}: {ops}"
|
||||
|
||||
|
||||
@then("the facade should be cached on subsequent calls")
|
||||
|
||||
@@ -361,7 +361,7 @@ def step_get_entry_count(context):
|
||||
context.entry_count = context.index.get_entry_count()
|
||||
|
||||
|
||||
@then("the count should be {count:d}")
|
||||
@then("idx the index count should be {count:d}")
|
||||
def step_check_entry_count_value(context, count):
|
||||
"""Verify the entry count matches the expected value."""
|
||||
assert context.entry_count == count
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch
|
||||
from behave import given, then, when
|
||||
|
||||
from cleveragents.cli.commands.actor import app as actor_app
|
||||
from cleveragents.core.exceptions import NotFoundError
|
||||
from cleveragents.domain.models.core.actor import Actor
|
||||
|
||||
|
||||
@@ -75,6 +76,22 @@ def step_impl(context: Any) -> None:
|
||||
_register_cleanup(context, context.actor_config_path)
|
||||
|
||||
|
||||
@given("I have an actor JSON config file with name {name}")
|
||||
def step_impl(context: Any, name: str) -> None:
|
||||
context.actor_config_data = {
|
||||
"name": name,
|
||||
"provider": "openai",
|
||||
"model": "gpt-4o-mini",
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(
|
||||
delete=False, suffix=".json", mode="w", encoding="utf-8"
|
||||
) as handle:
|
||||
json.dump(context.actor_config_data, handle)
|
||||
handle.flush()
|
||||
context.actor_config_path = Path(handle.name)
|
||||
_register_cleanup(context, context.actor_config_path)
|
||||
|
||||
|
||||
@when("I run actor add with NAME positional argument and config")
|
||||
def step_impl(context: Any) -> None:
|
||||
context.positional_name = "local/my-actor"
|
||||
@@ -117,8 +134,14 @@ def step_impl(context: Any) -> None:
|
||||
|
||||
@when("I run actor add with config but no NAME positional argument")
|
||||
def step_impl(context: Any) -> None:
|
||||
config_name = context.actor_config_data.get("name", "local/default-actor")
|
||||
mock_actor = _make_actor(name=config_name)
|
||||
with patch("cleveragents.cli.commands.actor._get_services") as mock_svc:
|
||||
registry = MagicMock()
|
||||
registry.get_actor.side_effect = NotFoundError(
|
||||
f"Actor not found: {config_name}"
|
||||
)
|
||||
registry.upsert_actor.return_value = mock_actor
|
||||
mock_svc.return_value = (MagicMock(), registry)
|
||||
context.result = context.runner.invoke(
|
||||
actor_app,
|
||||
@@ -128,6 +151,7 @@ def step_impl(context: Any) -> None:
|
||||
str(context.actor_config_path),
|
||||
],
|
||||
)
|
||||
context.mock_registry = registry
|
||||
|
||||
|
||||
@then("the actor add should succeed with the positional name")
|
||||
@@ -175,3 +199,34 @@ def step_impl(context: Any) -> None:
|
||||
f"Expected non-zero exit_code, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the actor add should succeed using the config name")
|
||||
def step_impl(context: Any) -> None:
|
||||
assert context.result.exit_code == 0, (
|
||||
f"Expected exit_code=0, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
assert context.mock_registry.upsert_actor.called, (
|
||||
"Expected upsert_actor to be called on the registry"
|
||||
)
|
||||
call_kwargs = context.mock_registry.upsert_actor.call_args
|
||||
actual_name = call_kwargs.kwargs.get("name") or (
|
||||
call_kwargs.args[0] if call_kwargs.args else None
|
||||
)
|
||||
expected_name = context.actor_config_data.get("name")
|
||||
assert actual_name == expected_name, (
|
||||
f"Expected upsert_actor called with name={expected_name!r} (from config), "
|
||||
f"got name={actual_name!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the actor add should fail with a BadParameter error about missing actor name")
|
||||
def step_impl(context: Any) -> None:
|
||||
assert context.result.exit_code != 0, (
|
||||
f"Expected non-zero exit_code, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
assert "Actor name is required" in context.result.output, (
|
||||
f"Expected 'Actor name is required' in output:\n{context.result.output}"
|
||||
)
|
||||
|
||||
@@ -107,7 +107,7 @@ def step_when_add_without_update(context: Any) -> None:
|
||||
mock_svc.return_value = (MagicMock(), registry)
|
||||
context.result = context.runner.invoke(
|
||||
actor_app,
|
||||
["add", "--config", str(context.actor_config_path)],
|
||||
["add", context.actor_name, "--config", str(context.actor_config_path)],
|
||||
)
|
||||
|
||||
|
||||
@@ -122,7 +122,13 @@ def step_when_add_with_update(context: Any) -> None:
|
||||
mock_svc.return_value = (MagicMock(), registry)
|
||||
context.result = context.runner.invoke(
|
||||
actor_app,
|
||||
["add", "--config", str(context.actor_config_path), "--update"],
|
||||
[
|
||||
"add",
|
||||
context.actor_name,
|
||||
"--config",
|
||||
str(context.actor_config_path),
|
||||
"--update",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -337,6 +337,63 @@ def step_remove_namespaced_ok(context: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
@when("I run actor remove with format json")
|
||||
def step_remove_format_json(context: Any) -> None:
|
||||
with (
|
||||
patch("cleveragents.cli.commands.actor._get_services") as mock_svc,
|
||||
patch("cleveragents.cli.commands.actor._compute_actor_impact") as mock_impact,
|
||||
):
|
||||
mock_registry = MagicMock()
|
||||
mock_service = MagicMock()
|
||||
actor = _make_actor(
|
||||
name="local/remove-json",
|
||||
provider="json-provider",
|
||||
model="gpt-json",
|
||||
)
|
||||
mock_registry.get_actor.return_value = actor
|
||||
mock_impact.return_value = (2, 1, 3)
|
||||
mock_svc.return_value = (mock_service, mock_registry)
|
||||
|
||||
context.result = context.runner.invoke(
|
||||
actor_app,
|
||||
["remove", actor.name, "--format", "json"],
|
||||
)
|
||||
|
||||
context.mock_actor_registry = mock_registry
|
||||
context.actor = actor
|
||||
context.impact_counts = (2, 1, 3)
|
||||
|
||||
|
||||
@then("the actor remove output should be valid JSON envelope")
|
||||
def step_remove_json_valid(context: Any) -> None:
|
||||
assert context.result.exit_code == 0
|
||||
parsed = json.loads(context.result.output.strip())
|
||||
assert _ENVELOPE_KEYS.issubset(parsed.keys())
|
||||
assert parsed["command"] == f"agents actor remove {context.actor.name}"
|
||||
assert parsed["status"] == "ok"
|
||||
assert parsed["exit_code"] == 0
|
||||
data = _unwrap_envelope(parsed)
|
||||
assert isinstance(data, dict)
|
||||
actor_data = data.get("actor_removed", {})
|
||||
assert actor_data.get("name") == context.actor.name
|
||||
assert actor_data.get("provider") == context.actor.provider
|
||||
assert actor_data.get("model") == context.actor.model
|
||||
impact = data.get("impact", {})
|
||||
expected_sessions, expected_plans, expected_actions = context.impact_counts
|
||||
assert impact.get("sessions") == expected_sessions
|
||||
assert impact.get("active_plans") == expected_plans
|
||||
assert impact.get("actions_referencing") == expected_actions
|
||||
cleanup = data.get("cleanup", {})
|
||||
assert cleanup.get("config") == "kept on disk"
|
||||
assert cleanup.get("contexts") == "0 orphaned"
|
||||
messages = parsed.get("messages", [])
|
||||
assert messages, "expected messages in envelope"
|
||||
first_message = messages[0]
|
||||
assert first_message.get("level") == "ok"
|
||||
assert "Actor removed" in first_message.get("text", "")
|
||||
context.mock_actor_registry.remove_actor.assert_called_once_with(context.actor.name)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Update with --format
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -135,7 +135,8 @@ def _subgraph_node(node_id: str, actor_ref: str = "") -> NodeDefinition:
|
||||
type=NodeType.SUBGRAPH,
|
||||
name=node_id.title(),
|
||||
description=f"Subgraph {node_id}",
|
||||
config={"actor_ref": actor_ref},
|
||||
config={},
|
||||
actor_ref=actor_ref if actor_ref else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -186,7 +186,8 @@ def step_given_subgraph_ref(context: Context, ref_name: str) -> None:
|
||||
type=NodeType.SUBGRAPH,
|
||||
name="Sub",
|
||||
description="Subgraph",
|
||||
config={"actor_ref": ref_name},
|
||||
config={},
|
||||
actor_ref=ref_name,
|
||||
),
|
||||
]
|
||||
edges = [EdgeDefinition(from_node="main", to_node="sub")]
|
||||
@@ -241,7 +242,8 @@ def step_given_outer_referencing_inner(
|
||||
type=NodeType.SUBGRAPH,
|
||||
name="Sub",
|
||||
description="Subgraph",
|
||||
config={"actor_ref": inner_name},
|
||||
config={},
|
||||
actor_ref=inner_name,
|
||||
),
|
||||
]
|
||||
edges = [EdgeDefinition(from_node="main", to_node="sub")]
|
||||
@@ -260,7 +262,8 @@ def step_given_resolver_cycle(context: Context, inner_name: str, back_ref: str)
|
||||
type=NodeType.SUBGRAPH,
|
||||
name="Child",
|
||||
description="Back-ref",
|
||||
config={"actor_ref": back_ref},
|
||||
config={},
|
||||
actor_ref=back_ref,
|
||||
),
|
||||
]
|
||||
inner = _build_graph_config(inner_name, inner_nodes, [], "child", ["child"])
|
||||
|
||||
@@ -17,11 +17,6 @@ from cleveragents.actor.yaml_loader import (
|
||||
interpolate_env_vars,
|
||||
load_yaml_text,
|
||||
)
|
||||
from cleveragents.actor.yaml_loader import (
|
||||
_restore_template_syntax,
|
||||
interpolate_env_vars,
|
||||
load_yaml_text,
|
||||
)
|
||||
|
||||
|
||||
@then('an actor config ValueError should mention "{text}"')
|
||||
@@ -314,76 +309,6 @@ def step_assert_options(context: Context) -> None:
|
||||
assert context.actor_cfg.options["temperature"] == 0.9
|
||||
|
||||
|
||||
@when("I extract v2 actor from a v2-style config with agents block")
|
||||
def step_extract_v2_actor(context: Context) -> None:
|
||||
data: dict[str, Any] = {
|
||||
"agents": {
|
||||
"main_agent": {
|
||||
"config": {
|
||||
"provider": "anthropic",
|
||||
"model": "claude-3",
|
||||
"unsafe": True,
|
||||
"options": {"max_tokens": 1000},
|
||||
}
|
||||
}
|
||||
},
|
||||
"routes": [{"from": "main_agent", "to": "end"}],
|
||||
}
|
||||
context.extracted = ActorConfiguration._extract_v2_actor(data)
|
||||
|
||||
|
||||
@then("the extracted provider and model should be set")
|
||||
def step_assert_extracted(context: Context) -> None:
|
||||
provider, model, _graph, unsafe = context.extracted
|
||||
assert provider == "anthropic"
|
||||
assert model == "claude-3"
|
||||
assert unsafe is True
|
||||
|
||||
|
||||
@then("the graph descriptor should contain agent info")
|
||||
def step_assert_graph_descriptor(context: Context) -> None:
|
||||
_, _, graph, _ = context.extracted
|
||||
assert graph is not None
|
||||
assert "agent" in graph
|
||||
assert "routes" in graph
|
||||
|
||||
|
||||
@when("I extract v2 actor from a config without agents block")
|
||||
def step_extract_v2_no_agents(context: Context) -> None:
|
||||
context.extracted = ActorConfiguration._extract_v2_actor({"provider": "openai"})
|
||||
|
||||
|
||||
@then("all extracted values should be None")
|
||||
def step_assert_all_none(context: Context) -> None:
|
||||
provider, model, graph, unsafe = context.extracted
|
||||
assert provider is None
|
||||
assert model is None
|
||||
assert graph is None
|
||||
assert unsafe is False
|
||||
|
||||
|
||||
@when("I extract v2 options from a v2-style config")
|
||||
def step_extract_v2_options(context: Context) -> None:
|
||||
data: dict[str, Any] = {
|
||||
"agents": {
|
||||
"main_agent": {
|
||||
"config": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4",
|
||||
"options": {"temperature": 0.7, "max_tokens": 500},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
context.options = ActorConfiguration._extract_v2_options(data)
|
||||
|
||||
|
||||
@then("the extracted options should contain expected keys")
|
||||
def step_assert_extracted_options(context: Context) -> None:
|
||||
assert context.options is not None
|
||||
assert "temperature" in context.options
|
||||
|
||||
|
||||
@when("I create an ActorConfiguration from the file")
|
||||
def step_from_file(context: Context) -> None:
|
||||
context.actor_cfg = ActorConfiguration.from_file(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,201 @@
|
||||
"""Step definitions for cross-actor subgraph cycle detection tests.
|
||||
|
||||
Tests for features/actor_subgraph_cycle_detection.feature — validates that
|
||||
the actor compiler correctly reads actor_ref from the top-level
|
||||
NodeDefinition field (not from node.config) when detecting cross-actor
|
||||
subgraph cycles.
|
||||
|
||||
This is the regression test for issue #1431: _detect_subgraph_cycles()
|
||||
was reading node.config.get("actor_ref", "") instead of node.actor_ref,
|
||||
causing cycle detection to always return an empty string and never detect
|
||||
cross-actor cycles.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.actor.compiler import (
|
||||
SubgraphCycleError,
|
||||
compile_actor,
|
||||
)
|
||||
from cleveragents.actor.schema import (
|
||||
ActorConfigSchema,
|
||||
ActorType,
|
||||
EdgeDefinition,
|
||||
NodeDefinition,
|
||||
NodeType,
|
||||
RouteDefinition,
|
||||
)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_graph_actor(
|
||||
name: str,
|
||||
nodes: list[NodeDefinition],
|
||||
edges: list[EdgeDefinition],
|
||||
entry_node: str,
|
||||
exit_nodes: list[str],
|
||||
) -> ActorConfigSchema:
|
||||
"""Build a minimal valid GRAPH actor for testing."""
|
||||
route = RouteDefinition(
|
||||
nodes=nodes,
|
||||
edges=edges,
|
||||
entry_node=entry_node,
|
||||
exit_nodes=exit_nodes,
|
||||
)
|
||||
return ActorConfigSchema(
|
||||
name=name,
|
||||
type=ActorType.GRAPH,
|
||||
description=f"Test actor {name}",
|
||||
provider="openai",
|
||||
model="gpt-4",
|
||||
route=route,
|
||||
)
|
||||
|
||||
|
||||
def _make_agent_node(node_id: str) -> NodeDefinition:
|
||||
"""Build a minimal AGENT node."""
|
||||
return NodeDefinition(
|
||||
id=node_id,
|
||||
type=NodeType.AGENT,
|
||||
name=node_id.title(),
|
||||
description=f"Agent {node_id}",
|
||||
config={"agent": "default"},
|
||||
)
|
||||
|
||||
|
||||
def _make_subgraph_node(node_id: str, actor_ref: str) -> NodeDefinition:
|
||||
"""Build a SUBGRAPH node using the top-level actor_ref field.
|
||||
|
||||
The actor_ref is set as a first-class field on NodeDefinition, NOT
|
||||
inside config. This is the correct way to reference a subgraph actor.
|
||||
"""
|
||||
return NodeDefinition(
|
||||
id=node_id,
|
||||
type=NodeType.SUBGRAPH,
|
||||
name=node_id.title(),
|
||||
description=f"Subgraph {node_id}",
|
||||
config={},
|
||||
actor_ref=actor_ref,
|
||||
)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Given steps
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@given("the actor compiler is available")
|
||||
def step_compiler_available(context: Context) -> None:
|
||||
"""Ensure the actor compiler module is importable."""
|
||||
context.actor_registry: dict[str, ActorConfigSchema] = {}
|
||||
|
||||
|
||||
@given('actor "{name}" has a subgraph node with actor_ref "{ref}"')
|
||||
def step_actor_has_subgraph_ref(context: Context, name: str, ref: str) -> None:
|
||||
"""Create a GRAPH actor with one agent node and one subgraph node."""
|
||||
agent = _make_agent_node("start")
|
||||
sub = _make_subgraph_node("sub", actor_ref=ref)
|
||||
actor = _make_graph_actor(
|
||||
name,
|
||||
nodes=[agent, sub],
|
||||
edges=[EdgeDefinition(from_node="start", to_node="sub")],
|
||||
entry_node="start",
|
||||
exit_nodes=["sub"],
|
||||
)
|
||||
context.actor_registry[name] = actor
|
||||
context.actor_to_compile = name
|
||||
|
||||
|
||||
@given('actor "{name}" has no subgraph nodes')
|
||||
def step_actor_has_no_subgraph(context: Context, name: str) -> None:
|
||||
"""Create a GRAPH actor with only an agent node (no subgraph references)."""
|
||||
agent = _make_agent_node("leaf")
|
||||
actor = _make_graph_actor(
|
||||
name,
|
||||
nodes=[agent],
|
||||
edges=[],
|
||||
entry_node="leaf",
|
||||
exit_nodes=["leaf"],
|
||||
)
|
||||
context.actor_registry[name] = actor
|
||||
|
||||
|
||||
@given("a registry containing both actors")
|
||||
def step_registry_contains_both(context: Context) -> None:
|
||||
"""Set up the resolver from the accumulated registry."""
|
||||
registry = context.actor_registry
|
||||
|
||||
def resolver(actor_name: str) -> ActorConfigSchema | None:
|
||||
return registry.get(actor_name)
|
||||
|
||||
context.resolver = resolver
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# When steps
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@when('I compile actor "{name}" with the registry resolver')
|
||||
def step_compile_actor(context: Context, name: str) -> None:
|
||||
"""Compile the named actor using the registry resolver."""
|
||||
context.compile_error = None
|
||||
context.compiled = None
|
||||
actor = context.actor_registry[name]
|
||||
try:
|
||||
context.compiled = compile_actor(actor, actor_resolver=context.resolver)
|
||||
except Exception as exc:
|
||||
context.compile_error = exc
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Then steps
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@then("the compilation should raise SubgraphCycleError")
|
||||
def step_raises_cycle_error(context: Context) -> None:
|
||||
"""Assert that compilation raised SubgraphCycleError."""
|
||||
assert context.compile_error is not None, (
|
||||
"Expected SubgraphCycleError but compilation succeeded"
|
||||
)
|
||||
assert isinstance(context.compile_error, SubgraphCycleError), (
|
||||
f"Expected SubgraphCycleError, got {type(context.compile_error).__name__}: "
|
||||
f"{context.compile_error}"
|
||||
)
|
||||
|
||||
|
||||
@then("the actor compilation should succeed")
|
||||
def step_actor_compilation_succeeds(context: Context) -> None:
|
||||
"""Assert that compilation succeeded without errors."""
|
||||
assert context.compile_error is None, (
|
||||
f"Expected compilation to succeed but got: {context.compile_error}"
|
||||
)
|
||||
assert context.compiled is not None
|
||||
|
||||
|
||||
@then('the cycle error message should mention "{text}"')
|
||||
def step_cycle_error_mentions(context: Context, text: str) -> None:
|
||||
"""Assert that the cycle error message contains the given text."""
|
||||
assert context.compile_error is not None
|
||||
msg = str(context.compile_error)
|
||||
assert text.lower() in msg.lower(), (
|
||||
f"Expected '{text}' in error message, got: {msg}"
|
||||
)
|
||||
|
||||
|
||||
@then('the actor subgraph_refs should map "{node}" to "{ref}"')
|
||||
def step_actor_subgraph_refs_map(context: Context, node: str, ref: str) -> None:
|
||||
"""Assert that the compiled metadata maps the node to the expected actor ref."""
|
||||
assert context.compiled is not None
|
||||
refs = context.compiled.metadata.subgraph_refs
|
||||
assert refs.get(node) == ref, (
|
||||
f"Expected subgraph_refs[{node!r}] == {ref!r}, got {refs}"
|
||||
)
|
||||
@@ -525,88 +525,6 @@ def step_validation_error_mapping(context: Any) -> None:
|
||||
assert "mapping" in msg, f"Error should mention 'mapping': {context.add_error}"
|
||||
|
||||
|
||||
@given("a legacy v3 YAML text with invalid schema")
|
||||
def step_legacy_v3_yaml_invalid_schema(context: Any) -> None:
|
||||
# is_v3_yaml detects version starting with "3" or non-null type field.
|
||||
# Provide a blob that triggers the v3 validation path but fails schema.
|
||||
context.legacy_v3_yaml_invalid = yaml.safe_dump(
|
||||
{
|
||||
"name": "local/bad-v3",
|
||||
"version": "3.0",
|
||||
"type": "invalid_type_value", # not llm/graph/tool — fails ActorConfigSchema
|
||||
"provider": "openai",
|
||||
"model": "gpt-4",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@when("I call ActorRegistry.add with the legacy invalid v3 YAML")
|
||||
def step_call_registry_add_legacy_invalid_v3(context: Any) -> None:
|
||||
try:
|
||||
context.registry.add(context.legacy_v3_yaml_invalid)
|
||||
context.add_error = None
|
||||
except ValidationError as exc:
|
||||
context.add_error = exc
|
||||
|
||||
|
||||
@then("a ValidationError should be raised mentioning invalid v3 actor")
|
||||
def step_validation_error_invalid_v3(context: Any) -> None:
|
||||
assert context.add_error is not None, "Expected a ValidationError to be raised"
|
||||
msg = str(context.add_error).lower()
|
||||
assert "invalid" in msg or "v3" in msg or "validation" in msg, (
|
||||
f"Error should mention invalid v3 actor: {context.add_error}"
|
||||
)
|
||||
|
||||
|
||||
@given("a legacy non-v3 YAML text missing provider and model")
|
||||
def step_legacy_non_v3_yaml_missing_fields(context: Any) -> None:
|
||||
context.legacy_non_v3_yaml = yaml.safe_dump(
|
||||
{
|
||||
"name": "local/no-provider",
|
||||
# no type field → not v3, no provider/model → should fail
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@when("I call ActorRegistry.add with the legacy non-v3 YAML")
|
||||
def step_call_registry_add_legacy_non_v3(context: Any) -> None:
|
||||
try:
|
||||
context.registry.add(context.legacy_non_v3_yaml)
|
||||
context.add_error = None
|
||||
except ValidationError as exc:
|
||||
context.add_error = exc
|
||||
|
||||
|
||||
@then("a ValidationError should be raised mentioning provider and model")
|
||||
def step_validation_error_provider_model(context: Any) -> None:
|
||||
assert context.add_error is not None, "Expected a ValidationError to be raised"
|
||||
msg = str(context.add_error).lower()
|
||||
assert "provider" in msg or "model" in msg, (
|
||||
f"Error should mention 'provider' or 'model': {context.add_error}"
|
||||
)
|
||||
|
||||
|
||||
@given("a legacy actor YAML text marked unsafe")
|
||||
def step_legacy_actor_yaml_unsafe(context: Any) -> None:
|
||||
context.legacy_unsafe_yaml = yaml.safe_dump(
|
||||
{
|
||||
"name": "local/legacy-unsafe",
|
||||
"provider": "openai",
|
||||
"model": "gpt-4",
|
||||
"unsafe": True,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@when("I call ActorRegistry.add with the legacy unsafe YAML")
|
||||
def step_call_registry_add_legacy_unsafe(context: Any) -> None:
|
||||
try:
|
||||
context.registry.add(context.legacy_unsafe_yaml)
|
||||
context.add_error = None
|
||||
except ValidationError as exc:
|
||||
context.add_error = exc
|
||||
|
||||
|
||||
@when("I call ActorRegistry.upsert_actor with an invalid v3 config_blob")
|
||||
def step_call_registry_upsert_invalid_v3(context: Any) -> None:
|
||||
# Provide a blob that triggers is_v3_yaml() but fails ActorConfigSchema.
|
||||
@@ -629,6 +547,15 @@ def step_call_registry_upsert_invalid_v3(context: Any) -> None:
|
||||
context.add_error = exc
|
||||
|
||||
|
||||
@then("a ValidationError should be raised mentioning invalid v3 actor")
|
||||
def step_validation_error_invalid_v3(context: Any) -> None:
|
||||
assert context.add_error is not None, "Expected a ValidationError to be raised"
|
||||
msg = str(context.add_error).lower()
|
||||
assert "invalid" in msg or "v3" in msg or "validation" in msg, (
|
||||
f"Error should mention invalid v3 actor: {context.add_error}"
|
||||
)
|
||||
|
||||
|
||||
# ── Coverage gap: config_parser.py ───────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -93,22 +93,6 @@ def step_v3_graph_blob(context: Any) -> None:
|
||||
}
|
||||
|
||||
|
||||
@given("a v2 actor blob with agents and routes")
|
||||
def step_v2_blob(context: Any) -> None:
|
||||
context.v2_blob = {
|
||||
"agents": {
|
||||
"main": {
|
||||
"type": "llm",
|
||||
"config": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4",
|
||||
},
|
||||
}
|
||||
},
|
||||
"routes": {},
|
||||
}
|
||||
|
||||
|
||||
@given("a mock actor service for v3 testing")
|
||||
def step_mock_actor_service(context: Any) -> None:
|
||||
mock_actor_service = MagicMock()
|
||||
@@ -247,13 +231,6 @@ def step_v3_graph_config_dict(context: Any) -> None:
|
||||
}
|
||||
|
||||
|
||||
@given("a v2 actor config dict with agents key")
|
||||
def step_v2_config_dict(context: Any) -> None:
|
||||
context.v2_config = {
|
||||
"agents": {"main": {"type": "llm", "config": {"provider": "openai"}}},
|
||||
}
|
||||
|
||||
|
||||
@given("an existing actor in the mock service")
|
||||
def step_existing_actor(context: Any) -> None:
|
||||
"""Set up mock to return an existing actor for duplicate checks."""
|
||||
@@ -274,14 +251,6 @@ def step_call_from_blob_v3(context: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
@when("I call ActorConfiguration.from_blob with the v2 blob")
|
||||
def step_call_from_blob_v2(context: Any) -> None:
|
||||
context.actor_config = ActorConfiguration.from_blob(
|
||||
blob=context.v2_blob,
|
||||
name="local/test",
|
||||
)
|
||||
|
||||
|
||||
@when("I call ActorRegistry.add with the v3 YAML")
|
||||
def step_call_registry_add_v3(context: Any) -> None:
|
||||
context.result_actor = context.registry.add(context.v3_yaml_text)
|
||||
@@ -329,11 +298,6 @@ def step_parse_v3_config(context: Any) -> None:
|
||||
context.reactive_config = parser._build(context.v3_config)
|
||||
|
||||
|
||||
@when("I check if the config is v3 format")
|
||||
def step_check_v3_format(context: Any) -> None:
|
||||
context.is_v3 = ReactiveConfigParser._is_v3_format(context.v2_config)
|
||||
|
||||
|
||||
# ── Then steps ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -470,11 +434,6 @@ def step_reactive_has_graph_route(context: Any) -> None:
|
||||
assert len(route.edges) >= 1, f"Expected at least 1 edge, got {len(route.edges)}"
|
||||
|
||||
|
||||
@then("it should not be detected as v3")
|
||||
def step_not_v3(context: Any) -> None:
|
||||
assert context.is_v3 is False, "Expected v2 data to NOT be detected as v3"
|
||||
|
||||
|
||||
@then("the graph route edges should use source and target keys")
|
||||
def step_graph_edges_use_source_target(context: Any) -> None:
|
||||
routes = context.reactive_config.routes
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Step definitions for behave_parallel_log_filtering.feature.
|
||||
|
||||
Tests the conditional log replay and worker exception handling in
|
||||
``scripts/run_behave_parallel.py``.
|
||||
``scripts/run_behave_parallel.py``, as well as the PassSuppressFormatter
|
||||
per-scenario output buffering behaviour.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,8 +13,11 @@ import sys
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when # type: ignore[import-untyped]
|
||||
from behave.configuration import Configuration # type: ignore[import-untyped]
|
||||
from behave.formatter.base import StreamOpener # type: ignore[import-untyped]
|
||||
from behave.runner import Context # type: ignore[import-untyped]
|
||||
|
||||
|
||||
@@ -52,6 +56,7 @@ _empty_summary = _runner_mod._empty_summary
|
||||
_worker_run_features = _runner_mod._worker_run_features
|
||||
_aggregate_worker_results = _runner_mod._aggregate_worker_results
|
||||
_has_failures = _runner_mod._has_failures
|
||||
PassSuppressFormatter = _runner_mod.PassSuppressFormatter
|
||||
|
||||
|
||||
def _passing_summary(features: int = 1, scenarios: int = 1) -> Summary:
|
||||
@@ -357,3 +362,132 @@ def step_worker_summary_features_errors_1(context: Context) -> None:
|
||||
f"Expected features.errors=1 so _has_failures() detects partial crash, "
|
||||
f"got: {errors}"
|
||||
)
|
||||
|
||||
|
||||
# ---- PassSuppressFormatter helpers ----
|
||||
|
||||
|
||||
class _MockStatus:
|
||||
"""Minimal status object mirroring behave's Status enum interface."""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
|
||||
|
||||
class _MockScenario:
|
||||
"""Minimal scenario object for PassSuppressFormatter unit tests."""
|
||||
|
||||
def __init__(self, name: str, status_name: str) -> None:
|
||||
self.keyword = "Scenario"
|
||||
self.name = name
|
||||
self.status = _MockStatus(status_name)
|
||||
|
||||
|
||||
class _MockStep:
|
||||
"""Minimal step object for PassSuppressFormatter unit tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
keyword: str,
|
||||
name: str,
|
||||
status_name: str,
|
||||
error_message: str | None,
|
||||
) -> None:
|
||||
self.keyword = keyword
|
||||
self.name = name
|
||||
self.status = _MockStatus(status_name)
|
||||
self.error_message = error_message
|
||||
|
||||
|
||||
def _make_pass_suppress_formatter() -> tuple[Any, io.StringIO]:
|
||||
"""Create a PassSuppressFormatter writing to a fresh StringIO buffer.
|
||||
|
||||
Returns ``(formatter, buffer)`` so callers can inspect what was written
|
||||
to the formatter's real output stream after simulating scenario events.
|
||||
"""
|
||||
buf: io.StringIO = io.StringIO()
|
||||
opener = StreamOpener(stream=buf)
|
||||
config = Configuration(command_args=["-q"])
|
||||
formatter: Any = PassSuppressFormatter(opener, config)
|
||||
return formatter, buf
|
||||
|
||||
|
||||
# ---- PassSuppressFormatter: Given ----
|
||||
|
||||
|
||||
@given("behave_parallel a PassSuppressFormatter backed by a captured stream")
|
||||
def step_create_pass_suppress_formatter(context: Context) -> None:
|
||||
formatter, buf = _make_pass_suppress_formatter()
|
||||
context.bp_formatter = formatter
|
||||
context.bp_formatter_stream = buf
|
||||
|
||||
|
||||
# ---- PassSuppressFormatter: When ----
|
||||
|
||||
|
||||
@when("behave_parallel I simulate a passing scenario through the formatter")
|
||||
def step_simulate_passing_scenario(context: Context) -> None:
|
||||
formatter: Any = context.bp_formatter
|
||||
scenario = _MockScenario("Passing scenario", "passed")
|
||||
step = _MockStep("Given", "I pass", "passed", None)
|
||||
formatter.scenario(scenario)
|
||||
formatter.result(step)
|
||||
formatter.eof()
|
||||
|
||||
|
||||
@when("behave_parallel I simulate a failing scenario through the formatter")
|
||||
def step_simulate_failing_scenario(context: Context) -> None:
|
||||
formatter: Any = context.bp_formatter
|
||||
scenario = _MockScenario("Failing scenario", "failed")
|
||||
step = _MockStep(
|
||||
"When",
|
||||
"I fail",
|
||||
"failed",
|
||||
"AssertionError: expected True got False",
|
||||
)
|
||||
formatter.scenario(scenario)
|
||||
formatter.result(step)
|
||||
formatter.eof()
|
||||
|
||||
|
||||
@when(
|
||||
"behave_parallel I simulate one passing then one failing scenario"
|
||||
" through the formatter"
|
||||
)
|
||||
def step_simulate_mixed_scenarios(context: Context) -> None:
|
||||
formatter: Any = context.bp_formatter
|
||||
# First: a passing scenario.
|
||||
passing_scenario = _MockScenario("Passing scenario", "passed")
|
||||
step1 = _MockStep("Given", "I pass", "passed", None)
|
||||
formatter.scenario(passing_scenario)
|
||||
formatter.result(step1)
|
||||
# Second: a failing scenario. Calling formatter.scenario() here finalises
|
||||
# the previous (passing) scenario, which should be discarded.
|
||||
failing_scenario = _MockScenario("Failing scenario", "failed")
|
||||
step2 = _MockStep("When", "I fail", "failed", "AssertionError: it broke")
|
||||
formatter.scenario(failing_scenario)
|
||||
formatter.result(step2)
|
||||
formatter.eof()
|
||||
|
||||
|
||||
# ---- PassSuppressFormatter: Then ----
|
||||
|
||||
|
||||
@then("behave_parallel the formatter real stream output should be empty")
|
||||
def step_formatter_output_empty(context: Context) -> None:
|
||||
output: str = context.bp_formatter_stream.getvalue()
|
||||
assert output == "", f"Expected empty output, got: {output!r}"
|
||||
|
||||
|
||||
@then('behave_parallel the formatter real stream output should contain "{text}"')
|
||||
def step_formatter_output_contains(context: Context, text: str) -> None:
|
||||
output: str = context.bp_formatter_stream.getvalue()
|
||||
assert text in output, f"Expected {text!r} in formatter output, got: {output!r}"
|
||||
|
||||
|
||||
@then('behave_parallel the formatter real stream output should not contain "{text}"')
|
||||
def step_formatter_output_not_contains(context: Context, text: str) -> None:
|
||||
output: str = context.bp_formatter_stream.getvalue()
|
||||
assert text not in output, (
|
||||
f"Expected {text!r} NOT in formatter output, got: {output!r}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Step definitions for CLI global options: --data-dir, --config-path, -v.
|
||||
|
||||
Tests for issue #6785 — spec-required global options that were missing from
|
||||
main_callback() and caused fatal "No such option" crashes.
|
||||
|
||||
All step patterns use the prefix "global options" to avoid any ambiguity
|
||||
with the many generic CLI step definitions in other step files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.formatting import OutputFormat
|
||||
from cleveragents.cli.main import app
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Regex to strip ANSI escape sequences (CSI sequences and OSC sequences).
|
||||
# Used to make output assertions environment-agnostic — Rich may emit ANSI
|
||||
# colour/style codes when the CI environment does not set TERM=dumb.
|
||||
_ANSI_ESCAPE_RE = re.compile(
|
||||
r"\x1b(?:"
|
||||
r"\[[0-9;]*[a-zA-Z]" # CSI sequences (SGR, cursor, etc.)
|
||||
r"|"
|
||||
r"\][^\x07]*\x07" # OSC sequences (hyperlink, title, etc.)
|
||||
r")"
|
||||
)
|
||||
|
||||
|
||||
def _strip_ansi(text: str) -> str:
|
||||
"""Remove ANSI escape sequences from *text*."""
|
||||
return _ANSI_ESCAPE_RE.sub("", text)
|
||||
|
||||
|
||||
def _invoke(args: list[str]) -> Any:
|
||||
"""Invoke the Typer app with the given args and return the result."""
|
||||
runner = CliRunner()
|
||||
return runner.invoke(app, args, catch_exceptions=True)
|
||||
|
||||
|
||||
def _combined_output(result: Any) -> str:
|
||||
"""Return combined output from a CliRunner result, with ANSI codes stripped.
|
||||
|
||||
In non-TTY environments (CI), Typer/Rich may split output between
|
||||
stdout and stderr. Check both explicitly so that ``--help`` output
|
||||
assertions work regardless of the terminal detection.
|
||||
"""
|
||||
stdout = _strip_ansi((getattr(result, "stdout", "") or "").strip())
|
||||
stderr = _strip_ansi((getattr(result, "stderr", "") or "").strip())
|
||||
combined = (
|
||||
(stdout + "\n" + stderr).strip() if (stdout and stderr) else (stdout or stderr)
|
||||
)
|
||||
return combined or _strip_ansi((result.output or "").strip())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("global options test env is clean")
|
||||
def step_global_opts_env_clean(context: Context) -> None:
|
||||
"""Ensure test-specific env vars are not polluted from previous tests.
|
||||
|
||||
Registers a cleanup handler that restores the original env var values after
|
||||
the scenario ends. The global ``after_scenario`` hook in
|
||||
``features/environment.py`` runs all ``context._cleanup_handlers``.
|
||||
"""
|
||||
orig_data_dir = os.environ.get("CLEVERAGENTS_DATA_DIR")
|
||||
orig_config_path = os.environ.get("CLEVERAGENTS_CONFIG_PATH")
|
||||
# Clean up any leftover from previous scenarios
|
||||
for var in ("CLEVERAGENTS_DATA_DIR", "CLEVERAGENTS_CONFIG_PATH"):
|
||||
os.environ.pop(var, None)
|
||||
|
||||
# Register restoration handler so that env vars set by main_callback()
|
||||
# during CLI invocation do not pollute subsequent scenarios.
|
||||
def _restore() -> None:
|
||||
for key, val in [
|
||||
("CLEVERAGENTS_DATA_DIR", orig_data_dir),
|
||||
("CLEVERAGENTS_CONFIG_PATH", orig_config_path),
|
||||
]:
|
||||
if val is not None:
|
||||
os.environ[key] = val
|
||||
else:
|
||||
os.environ.pop(key, None)
|
||||
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(_restore)
|
||||
|
||||
|
||||
@given("a temp data dir is prepared")
|
||||
def step_temp_data_dir_prepared(context: Context) -> None:
|
||||
"""Create a real temporary directory for --data-dir testing.
|
||||
|
||||
Registers a cleanup handler so the directory is removed after the scenario.
|
||||
"""
|
||||
tmp_dir = tempfile.mkdtemp(prefix="ca_test_data_dir_")
|
||||
context.temp_data_dir = tmp_dir
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: shutil.rmtree(tmp_dir, ignore_errors=True))
|
||||
|
||||
|
||||
@given("a temp config file is prepared")
|
||||
def step_temp_config_file_prepared(context: Context) -> None:
|
||||
"""Create a real temporary TOML config file for --config-path testing.
|
||||
|
||||
Registers a cleanup handler so the file is removed after the scenario.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".toml", prefix="ca_test_config_", delete=False
|
||||
) as tmp_file:
|
||||
tmp_file.write(b"# test config\n")
|
||||
tmp_path = tmp_file.name
|
||||
context.temp_config_file = tmp_path
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: os.unlink(tmp_path))
|
||||
|
||||
|
||||
@given("a temp file path is prepared")
|
||||
def step_temp_file_path_prepared(context: Context) -> None:
|
||||
"""Create a temporary FILE (not directory) to trigger --data-dir validation.
|
||||
|
||||
Registers a cleanup handler so the file is removed after the scenario.
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(prefix="ca_test_file_", delete=False) as tmp_file:
|
||||
tmp_path = tmp_file.name
|
||||
context.temp_file_path = tmp_path
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
context._cleanup_handlers.append(lambda: os.unlink(tmp_path))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps - CLI invocation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I run the global options CLI with data-dir flag and "{subcommand}"')
|
||||
def step_run_global_opts_data_dir(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with --data-dir pointing at the temp directory."""
|
||||
args = ["--data-dir", context.temp_data_dir, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with config-path flag and "{subcommand}"')
|
||||
def step_run_global_opts_config_path(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with --config-path pointing at the temp config file."""
|
||||
args = ["--config-path", context.temp_config_file, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with both path flags and "{subcommand}"')
|
||||
def step_run_global_opts_both(context: Context, subcommand: str) -> None:
|
||||
"""Run the CLI with both --data-dir and --config-path."""
|
||||
args = [
|
||||
"--data-dir",
|
||||
context.temp_data_dir,
|
||||
"--config-path",
|
||||
context.temp_config_file,
|
||||
subcommand,
|
||||
]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with data-dir as temp file and "{subcommand}"')
|
||||
def step_run_global_opts_data_dir_file(context: Context, subcommand: str) -> None:
|
||||
"""Run CLI with --data-dir pointing to a FILE (triggers validation error)."""
|
||||
args = ["--data-dir", context.temp_file_path, subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with invalid config-path and "{subcommand}"')
|
||||
def step_run_global_opts_invalid_config_path(context: Context, subcommand: str) -> None:
|
||||
"""Run CLI with --config-path pointing to a non-existent file."""
|
||||
args = ["--config-path", "/tmp/no_such_file_xyz_99999.toml", subcommand]
|
||||
result = _invoke(args)
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
@when('I run the global options CLI with args "{args}"')
|
||||
def step_run_global_opts_with_args(context: Context, args: str) -> None:
|
||||
"""Run the CLI with space-separated args string."""
|
||||
result = _invoke(args.split())
|
||||
context.global_opts_result = result
|
||||
context.global_opts_exit_code = result.exit_code
|
||||
context.global_opts_output = _combined_output(result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps - direct callback testing for verbosity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I call main_callback with verbosity {count:d}")
|
||||
def step_call_main_callback_verbosity(context: Context, count: int) -> None:
|
||||
"""Invoke main_callback directly, capturing the log level configured."""
|
||||
from cleveragents.cli.main import main_callback
|
||||
|
||||
context.global_opts_captured_log_level: str | None = None
|
||||
context.global_opts_captured_ctx_obj: dict[str, Any] = {}
|
||||
|
||||
def _capture_log_level(
|
||||
*, env: str = "development", log_level: str = "INFO"
|
||||
) -> None:
|
||||
context.global_opts_captured_log_level = log_level
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx.obj = {}
|
||||
|
||||
with patch(
|
||||
"cleveragents.config.logging.configure_structlog",
|
||||
side_effect=_capture_log_level,
|
||||
):
|
||||
main_callback(
|
||||
ctx=mock_ctx,
|
||||
version=None, # type: ignore[arg-type]
|
||||
show_secrets=False,
|
||||
fmt=OutputFormat.RICH,
|
||||
data_dir=None,
|
||||
config_path=None,
|
||||
verbose=count,
|
||||
)
|
||||
|
||||
context.global_opts_captured_ctx_obj = mock_ctx.obj
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - exit codes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the global options CLI should succeed")
|
||||
def step_global_opts_cli_succeed(context: Context) -> None:
|
||||
actual = context.global_opts_exit_code
|
||||
output = context.global_opts_output
|
||||
assert actual == 0, f"Expected exit code 0, got {actual}.\nOutput:\n{output}"
|
||||
|
||||
|
||||
@then("the global options CLI should fail")
|
||||
def step_global_opts_cli_fail(context: Context) -> None:
|
||||
actual = context.global_opts_exit_code
|
||||
assert actual != 0, (
|
||||
f"Expected non-zero exit code, got {actual}.\nOutput:\n{context.global_opts_output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - env var assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the CLEVERAGENTS_DATA_DIR env var should match the temp data dir")
|
||||
def step_env_data_dir_matches(context: Context) -> None:
|
||||
"""Verify that --data-dir set CLEVERAGENTS_DATA_DIR to the temp directory."""
|
||||
env_val = os.environ.get("CLEVERAGENTS_DATA_DIR")
|
||||
expected = str(Path(context.temp_data_dir).resolve())
|
||||
assert env_val is not None, (
|
||||
"CLEVERAGENTS_DATA_DIR was not set after --data-dir invocation"
|
||||
)
|
||||
assert str(Path(env_val).resolve()) == expected, (
|
||||
f"CLEVERAGENTS_DATA_DIR={env_val!r} does not match expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the CLEVERAGENTS_CONFIG_PATH env var should match the temp config file")
|
||||
def step_env_config_path_matches(context: Context) -> None:
|
||||
"""Verify that --config-path set CLEVERAGENTS_CONFIG_PATH to the temp file."""
|
||||
env_val = os.environ.get("CLEVERAGENTS_CONFIG_PATH")
|
||||
expected = str(Path(context.temp_config_file).resolve())
|
||||
assert env_val is not None, (
|
||||
"CLEVERAGENTS_CONFIG_PATH was not set after --config-path invocation"
|
||||
)
|
||||
assert str(Path(env_val).resolve()) == expected, (
|
||||
f"CLEVERAGENTS_CONFIG_PATH={env_val!r} does not match expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - ctx.obj assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options ctx obj "{key}" should be {value:d}')
|
||||
def step_global_opts_ctx_obj_int(context: Context, key: str, value: int) -> None:
|
||||
"""Verify ctx.obj[key] has the expected integer value."""
|
||||
obj = context.global_opts_captured_ctx_obj
|
||||
assert key in obj, f"ctx.obj missing key '{key}'. ctx.obj={obj}"
|
||||
actual = obj[key]
|
||||
assert actual == value, f"ctx.obj['{key}']={actual!r}, expected {value!r}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - log level assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options log level should be "{level}"')
|
||||
def step_global_opts_log_level(context: Context, level: str) -> None:
|
||||
"""Verify that configure_structlog was called with the expected log level."""
|
||||
actual = context.global_opts_captured_log_level
|
||||
assert actual is not None, "configure_structlog was not called"
|
||||
assert actual.upper() == level.upper(), (
|
||||
f"Expected log level {level!r}, got {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - output assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('the global options output should contain "{text}"')
|
||||
def step_global_opts_output_contains(context: Context, text: str) -> None:
|
||||
output = context.global_opts_output
|
||||
assert text in output, f"Expected '{text}' in output:\n{output}"
|
||||
@@ -27,7 +27,8 @@ def _restore_cwd(context):
|
||||
os.environ.pop("CLEVERAGENTS_HOME", None)
|
||||
else:
|
||||
os.environ["CLEVERAGENTS_HOME"] = context._init_original_home
|
||||
shutil.rmtree(context.temp_dir, ignore_errors=True)
|
||||
if getattr(context, "temp_dir", None) is not None:
|
||||
shutil.rmtree(context.temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _create_init_mocks(context):
|
||||
|
||||
@@ -1105,3 +1105,146 @@ def step_try_store_duplicate(context: Context) -> None:
|
||||
def step_duplicate_error_raised(context: Context) -> None:
|
||||
assert context.decision_error is not None
|
||||
assert isinstance(context.decision_error, DuplicateDecisionError)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full context snapshot steps (issue #9056)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a plan lifecycle service with decision service wired")
|
||||
def step_lifecycle_with_decision_service(context: Context) -> None:
|
||||
"""Create a PlanLifecycleService with a real DecisionService wired in."""
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.application.services.plan_lifecycle_service import (
|
||||
PlanLifecycleService,
|
||||
)
|
||||
from cleveragents.config.settings import Settings
|
||||
|
||||
Settings._instance = None
|
||||
settings = Settings()
|
||||
context.snapshot_decision_service = DecisionService()
|
||||
context.snapshot_lifecycle_service = PlanLifecycleService(
|
||||
settings=settings,
|
||||
decision_service=context.snapshot_decision_service,
|
||||
)
|
||||
context.snapshot_plan = None
|
||||
context.snapshot_decision = None
|
||||
context.snapshot_action_name = None
|
||||
|
||||
|
||||
@given('an action "{action_name}" for strategize snapshot test')
|
||||
def step_create_action_for_snapshot_test(context: Context, action_name: str) -> None:
|
||||
"""Create an action for the strategize snapshot test."""
|
||||
context.snapshot_action_name = action_name
|
||||
context.snapshot_lifecycle_service.create_action(
|
||||
name=action_name,
|
||||
description=f"Action {action_name} for snapshot test",
|
||||
definition_of_done="Snapshot test done",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
)
|
||||
|
||||
|
||||
@given('a plan created from "{action_name}" with project "{project_name}"')
|
||||
def step_create_plan_with_project(
|
||||
context: Context, action_name: str, project_name: str
|
||||
) -> None:
|
||||
"""Create a plan from the given action with a project link."""
|
||||
from cleveragents.domain.models.core.plan import ProjectLink
|
||||
|
||||
context.snapshot_plan = context.snapshot_lifecycle_service.use_action(
|
||||
action_name=action_name,
|
||||
project_links=[ProjectLink(project_name=project_name)],
|
||||
)
|
||||
|
||||
|
||||
@given('a plan created from "{action_name}" without projects')
|
||||
def step_create_plan_without_projects(context: Context, action_name: str) -> None:
|
||||
"""Create a plan from the given action without any project links."""
|
||||
context.snapshot_plan = context.snapshot_lifecycle_service.use_action(
|
||||
action_name=action_name,
|
||||
project_links=[],
|
||||
)
|
||||
|
||||
|
||||
@when("I start strategize for the snapshot test plan")
|
||||
def step_start_strategize_snapshot_test(context: Context) -> None:
|
||||
"""Start strategize and capture the recorded decision."""
|
||||
plan_id = context.snapshot_plan.identity.plan_id
|
||||
context.snapshot_lifecycle_service.start_strategize(plan_id)
|
||||
|
||||
decisions = context.snapshot_decision_service.list_decisions(plan_id)
|
||||
assert len(decisions) >= 1, f"Expected at least 1 decision, got {len(decisions)}"
|
||||
strategy_decisions = [
|
||||
d for d in decisions if d.decision_type.value == "strategy_choice"
|
||||
]
|
||||
assert len(strategy_decisions) >= 1, "Expected at least 1 strategy_choice decision"
|
||||
context.snapshot_decision = strategy_decisions[0]
|
||||
|
||||
|
||||
@then("the strategize decision should have a non-empty hot_context_hash")
|
||||
def step_check_snapshot_hash_not_empty(context: Context) -> None:
|
||||
"""Verify the context snapshot hash is not empty."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert snapshot.hot_context_hash, "hot_context_hash should not be empty"
|
||||
|
||||
|
||||
@then("the strategize decision should have a non-empty hot_context_ref")
|
||||
def step_check_snapshot_ref_not_empty(context: Context) -> None:
|
||||
"""Verify the context snapshot ref is not empty."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert snapshot.hot_context_ref, "hot_context_ref should not be empty"
|
||||
|
||||
|
||||
@then('the strategize decision hot_context_ref should start with "{prefix}"')
|
||||
def step_check_snapshot_ref_prefix(context: Context, prefix: str) -> None:
|
||||
"""Verify the context snapshot ref starts with the expected prefix."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert snapshot.hot_context_ref.startswith(prefix), (
|
||||
f"hot_context_ref should start with {prefix!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the strategize decision should have a non-empty actor_state_ref")
|
||||
def step_check_snapshot_actor_ref_not_empty(context: Context) -> None:
|
||||
"""Verify the actor_state_ref is not empty."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert snapshot.actor_state_ref, "actor_state_ref should not be empty"
|
||||
|
||||
|
||||
@then("the strategize decision should have relevant_resources populated")
|
||||
def step_check_snapshot_resources_populated(context: Context) -> None:
|
||||
"""Verify relevant_resources is not empty."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert len(snapshot.relevant_resources) > 0, (
|
||||
"relevant_resources should not be empty"
|
||||
)
|
||||
|
||||
|
||||
@then('the strategize decision hot_context_hash should start with "{prefix}"')
|
||||
def step_check_snapshot_hash_prefix(context: Context, prefix: str) -> None:
|
||||
"""Verify the context snapshot hash starts with the expected prefix."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert snapshot.hot_context_hash.startswith(prefix), (
|
||||
f"hot_context_hash should start with {prefix!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the strategize decision hot_context_hash should be {length:d} characters long")
|
||||
def step_check_snapshot_hash_length(context: Context, length: int) -> None:
|
||||
"""Verify the context snapshot hash has the expected length."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
actual_length = len(snapshot.hot_context_hash)
|
||||
assert actual_length == length, (
|
||||
f"hot_context_hash length should be {length}, got {actual_length}"
|
||||
)
|
||||
|
||||
|
||||
@then("the strategize decision should have empty relevant_resources")
|
||||
def step_check_snapshot_resources_empty(context: Context) -> None:
|
||||
"""Verify relevant_resources is empty."""
|
||||
snapshot = context.snapshot_decision.context_snapshot
|
||||
assert len(snapshot.relevant_resources) == 0, (
|
||||
f"relevant_resources should be empty, got {snapshot.relevant_resources}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Step definitions for devcontainer_autodiscovery_wiring.feature.
|
||||
|
||||
Tests that discover_devcontainers() is correctly wired into
|
||||
GitCheckoutHandler.discover_children() and FsDirectoryHandler.discover_children().
|
||||
|
||||
All steps use the ``dcwire`` prefix to avoid Behave AmbiguousStep errors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
from cleveragents.domain.models.core.resource import PhysVirt, Resource
|
||||
from cleveragents.resource.handlers.fs_directory import FsDirectoryHandler
|
||||
from cleveragents.resource.handlers.git_checkout import GitCheckoutHandler
|
||||
|
||||
_VALID_ULID = "01JQDVHN5X5QJKBMZ3AP8Y4G7K"
|
||||
_DEVCONTAINER_JSON = json.dumps({"name": "Test Dev Container", "image": "ubuntu:22.04"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_git_resource(location: str) -> Resource:
|
||||
"""Create a minimal git-checkout Resource."""
|
||||
return Resource(
|
||||
resource_id=_VALID_ULID,
|
||||
name="test-repo",
|
||||
resource_type_name="git-checkout",
|
||||
classification=PhysVirt.PHYSICAL,
|
||||
description="Test git checkout resource",
|
||||
location=location,
|
||||
)
|
||||
|
||||
|
||||
def _make_fs_resource(location: str) -> Resource:
|
||||
"""Create a minimal fs-directory Resource."""
|
||||
return Resource(
|
||||
resource_id=_VALID_ULID,
|
||||
name="test-dir",
|
||||
resource_type_name="fs-directory",
|
||||
classification=PhysVirt.PHYSICAL,
|
||||
description="Test filesystem directory resource",
|
||||
location=location,
|
||||
)
|
||||
|
||||
|
||||
def _init_git_repo(tmpdir: str, extra_files: dict[str, str] | None = None) -> str:
|
||||
"""Initialise a bare-minimum git repo with an initial commit."""
|
||||
subprocess.run(["git", "init"], cwd=tmpdir, capture_output=True, check=True)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "test@dcwire.dev"],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "DCWire Test"],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "commit.gpgSign", "false"],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
# Always create a README so there is at least one tracked file
|
||||
readme = Path(tmpdir) / "README.md"
|
||||
readme.write_text("# dcwire test\n")
|
||||
|
||||
if extra_files:
|
||||
for rel_path, file_content in extra_files.items():
|
||||
full = Path(tmpdir) / rel_path
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
full.write_text(file_content)
|
||||
|
||||
subprocess.run(["git", "add", "."], cwd=tmpdir, capture_output=True, check=True)
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", "init"],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return tmpdir
|
||||
|
||||
|
||||
def _create_devcontainer_dir(base: str, rel_path: str) -> None:
|
||||
"""Create a devcontainer.json at the given relative path inside base."""
|
||||
full = Path(base) / rel_path
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
full.write_text(_DEVCONTAINER_JSON)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GIVEN steps — git-checkout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('dcwire a git repo with a ".devcontainer/devcontainer.json" file')
|
||||
def step_dcwire_git_with_root_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-git-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/devcontainer.json")
|
||||
_init_git_repo(tmpdir, {".devcontainer/devcontainer.json": _DEVCONTAINER_JSON})
|
||||
ctx.dcwire_handler = GitCheckoutHandler()
|
||||
ctx.dcwire_resource = _make_git_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given('dcwire a git repo with a ".devcontainer/api/devcontainer.json" named config')
|
||||
def step_dcwire_git_with_named_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-git-named-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/api/devcontainer.json")
|
||||
_init_git_repo(tmpdir, {".devcontainer/api/devcontainer.json": _DEVCONTAINER_JSON})
|
||||
ctx.dcwire_handler = GitCheckoutHandler()
|
||||
ctx.dcwire_resource = _make_git_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given("dcwire a git repo with no devcontainer configuration")
|
||||
def step_dcwire_git_no_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-git-nodc-")
|
||||
_init_git_repo(tmpdir)
|
||||
ctx.dcwire_handler = GitCheckoutHandler()
|
||||
ctx.dcwire_resource = _make_git_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given(
|
||||
'dcwire a git repo with a subdirectory "src" and a ".devcontainer/devcontainer.json" file'
|
||||
)
|
||||
def step_dcwire_git_with_src_and_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-git-src-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/devcontainer.json")
|
||||
_init_git_repo(
|
||||
tmpdir,
|
||||
{
|
||||
"src/main.py": "print('hello')\n",
|
||||
".devcontainer/devcontainer.json": _DEVCONTAINER_JSON,
|
||||
},
|
||||
)
|
||||
ctx.dcwire_handler = GitCheckoutHandler()
|
||||
ctx.dcwire_resource = _make_git_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given('dcwire a git repo with a root-level ".devcontainer.json" file')
|
||||
def step_dcwire_git_with_root_level_devcontainer_json(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-git-rootdc-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer.json")
|
||||
_init_git_repo(tmpdir, {".devcontainer.json": _DEVCONTAINER_JSON})
|
||||
ctx.dcwire_handler = GitCheckoutHandler()
|
||||
ctx.dcwire_resource = _make_git_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GIVEN steps — fs-directory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('dcwire a filesystem directory with a ".devcontainer/devcontainer.json" file')
|
||||
def step_dcwire_fs_with_root_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-fs-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/devcontainer.json")
|
||||
ctx.dcwire_handler = FsDirectoryHandler()
|
||||
ctx.dcwire_resource = _make_fs_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given(
|
||||
'dcwire a filesystem directory with a ".devcontainer/frontend/devcontainer.json" named config'
|
||||
)
|
||||
def step_dcwire_fs_with_named_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-fs-named-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/frontend/devcontainer.json")
|
||||
ctx.dcwire_handler = FsDirectoryHandler()
|
||||
ctx.dcwire_resource = _make_fs_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given("dcwire a filesystem directory with no devcontainer configuration")
|
||||
def step_dcwire_fs_no_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-fs-nodc-")
|
||||
ctx.dcwire_handler = FsDirectoryHandler()
|
||||
ctx.dcwire_resource = _make_fs_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given(
|
||||
'dcwire a filesystem directory with a subdirectory "lib" and a ".devcontainer/devcontainer.json" file'
|
||||
)
|
||||
def step_dcwire_fs_with_lib_and_devcontainer(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-fs-lib-")
|
||||
lib_dir = Path(tmpdir) / "lib"
|
||||
lib_dir.mkdir()
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer/devcontainer.json")
|
||||
ctx.dcwire_handler = FsDirectoryHandler()
|
||||
ctx.dcwire_resource = _make_fs_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
@given('dcwire a filesystem directory with a root-level ".devcontainer.json" file')
|
||||
def step_dcwire_fs_with_root_level_devcontainer_json(ctx):
|
||||
tmpdir = tempfile.mkdtemp(prefix="dcwire-fs-rootdc-")
|
||||
_create_devcontainer_dir(tmpdir, ".devcontainer.json")
|
||||
ctx.dcwire_handler = FsDirectoryHandler()
|
||||
ctx.dcwire_resource = _make_fs_resource(tmpdir)
|
||||
ctx.dcwire_result = None
|
||||
ctx.dcwire_error = None
|
||||
ctx.dcwire_last_dc_child = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WHEN steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("dcwire I call discover_children on the git-checkout resource")
|
||||
def step_dcwire_discover_git(ctx):
|
||||
try:
|
||||
ctx.dcwire_result = ctx.dcwire_handler.discover_children(
|
||||
resource=ctx.dcwire_resource
|
||||
)
|
||||
except Exception as exc:
|
||||
ctx.dcwire_error = exc
|
||||
|
||||
|
||||
@when("dcwire I call discover_children on the fs-directory resource")
|
||||
def step_dcwire_discover_fs(ctx):
|
||||
try:
|
||||
ctx.dcwire_result = ctx.dcwire_handler.discover_children(
|
||||
resource=ctx.dcwire_resource
|
||||
)
|
||||
except Exception as exc:
|
||||
ctx.dcwire_error = exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# THEN steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then('dcwire the children include a "devcontainer-instance" resource named "{name}"')
|
||||
def step_dcwire_has_devcontainer_child(ctx, name):
|
||||
assert ctx.dcwire_error is None, f"Unexpected error: {ctx.dcwire_error}"
|
||||
assert ctx.dcwire_result is not None, "Expected a list of children"
|
||||
dc_children = [
|
||||
r
|
||||
for r in ctx.dcwire_result
|
||||
if r.resource_type_name == "devcontainer-instance" and r.name == name
|
||||
]
|
||||
assert len(dc_children) == 1, (
|
||||
f"Expected exactly one devcontainer-instance named '{name}', "
|
||||
f"got {[r.name for r in ctx.dcwire_result if r.resource_type_name == 'devcontainer-instance']}"
|
||||
)
|
||||
ctx.dcwire_last_dc_child = dc_children[0]
|
||||
|
||||
|
||||
@then('dcwire the children include a "fs-directory" resource named "{name}"')
|
||||
def step_dcwire_has_fs_child(ctx, name):
|
||||
assert ctx.dcwire_error is None, f"Unexpected error: {ctx.dcwire_error}"
|
||||
assert ctx.dcwire_result is not None, "Expected a list of children"
|
||||
fs_children = [
|
||||
r
|
||||
for r in ctx.dcwire_result
|
||||
if r.resource_type_name == "fs-directory" and r.name == name
|
||||
]
|
||||
assert len(fs_children) == 1, (
|
||||
f"Expected exactly one fs-directory named '{name}', "
|
||||
f"got {[r.name for r in ctx.dcwire_result if r.resource_type_name == 'fs-directory']}"
|
||||
)
|
||||
|
||||
|
||||
@then('dcwire the devcontainer child has provisioning_state "{state}"')
|
||||
def step_dcwire_has_provisioning_state(ctx, state):
|
||||
assert ctx.dcwire_last_dc_child is not None, "No devcontainer child captured"
|
||||
props = ctx.dcwire_last_dc_child.properties or {}
|
||||
actual = props.get("provisioning_state")
|
||||
assert actual == state, f"Expected provisioning_state='{state}', got '{actual}'"
|
||||
|
||||
|
||||
@then("dcwire the devcontainer child has devcontainer_json_path set")
|
||||
def step_dcwire_has_json_path(ctx):
|
||||
assert ctx.dcwire_last_dc_child is not None, "No devcontainer child captured"
|
||||
props = ctx.dcwire_last_dc_child.properties or {}
|
||||
path_val = props.get("devcontainer_json_path")
|
||||
assert path_val, f"Expected devcontainer_json_path to be set, got {path_val!r}"
|
||||
assert "devcontainer.json" in path_val, (
|
||||
f"Expected devcontainer_json_path to contain 'devcontainer.json', got {path_val!r}"
|
||||
)
|
||||
|
||||
|
||||
@then('dcwire the devcontainer child has config_name "{config_name}"')
|
||||
def step_dcwire_has_config_name(ctx, config_name):
|
||||
assert ctx.dcwire_last_dc_child is not None, "No devcontainer child captured"
|
||||
props = ctx.dcwire_last_dc_child.properties or {}
|
||||
actual = props.get("config_name")
|
||||
assert actual == config_name, (
|
||||
f"Expected config_name='{config_name}', got '{actual}'"
|
||||
)
|
||||
|
||||
|
||||
@then("dcwire no devcontainer-instance children are present")
|
||||
def step_dcwire_no_devcontainer_children(ctx):
|
||||
assert ctx.dcwire_error is None, f"Unexpected error: {ctx.dcwire_error}"
|
||||
assert ctx.dcwire_result is not None, "Expected a list of children"
|
||||
dc_children = [
|
||||
r for r in ctx.dcwire_result if r.resource_type_name == "devcontainer-instance"
|
||||
]
|
||||
assert len(dc_children) == 0, (
|
||||
f"Expected no devcontainer-instance children, got {[r.name for r in dc_children]}"
|
||||
)
|
||||
@@ -755,7 +755,7 @@ def step_try_create_plan_empty_desc(context: Context) -> None:
|
||||
context.pydantic_error = exc
|
||||
|
||||
|
||||
@then("a Pydantic validation error should be raised")
|
||||
@then("an Edge Case Pydantic validation error should be raised")
|
||||
def step_check_pydantic_error(context: Context) -> None:
|
||||
assert context.pydantic_error is not None, "Expected a Pydantic validation error"
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ def step_attempt_rollback(context):
|
||||
)
|
||||
|
||||
|
||||
@then("the rollback result should be False")
|
||||
@then("the executor rollback result should be False")
|
||||
def step_verify_rollback_false(context):
|
||||
"""Verify the rollback result is False."""
|
||||
assert context.rollback_result is False
|
||||
|
||||
@@ -807,6 +807,22 @@ def step_pec_output_valid_json_list(context: Context) -> None:
|
||||
assert isinstance(data, list), "Expected JSON array"
|
||||
|
||||
|
||||
@then("pec the output should be valid json envelope")
|
||||
def step_pec_output_valid_json_envelope(context: Context) -> None:
|
||||
parsed = json.loads(context.pec_result.output.strip())
|
||||
assert isinstance(parsed, dict), f"Expected JSON object, got {type(parsed)}"
|
||||
assert _ENVELOPE_KEYS.issubset(parsed.keys()), (
|
||||
f"Expected envelope keys {_ENVELOPE_KEYS}, got {set(parsed.keys())}"
|
||||
)
|
||||
data = parsed["data"]
|
||||
assert isinstance(data, dict), (
|
||||
f"Expected envelope data to be a dict, got {type(data)}"
|
||||
)
|
||||
assert "plan_id" in data, (
|
||||
f"Expected 'plan_id' in envelope data, got {set(data.keys())}"
|
||||
)
|
||||
|
||||
|
||||
@then("pec the tree should exclude the superseded grandchild")
|
||||
def step_pec_tree_excludes_orphan(context: Context) -> None:
|
||||
# Tree should have exactly one root with one child and zero grandchildren.
|
||||
|
||||
@@ -331,7 +331,7 @@ def step_json_contains(context: Context, text: str) -> None:
|
||||
assert text in context.pe_json_output, f"Expected '{text}' in JSON output"
|
||||
|
||||
|
||||
@then("the json output should be valid json")
|
||||
@then("the plan explain json output should be valid")
|
||||
def step_json_valid(context: Context) -> None:
|
||||
parsed = json.loads(context.pe_json_output)
|
||||
assert isinstance(parsed, dict), "Expected a JSON object"
|
||||
@@ -405,6 +405,18 @@ def step_tree_json_valid(context: Context) -> None:
|
||||
assert isinstance(parsed, list), "Expected a JSON array"
|
||||
|
||||
|
||||
@then("the json tree output should be a valid json envelope")
|
||||
def step_tree_json_valid_envelope(context: Context) -> None:
|
||||
parsed = json.loads(context.pe_tree_json)
|
||||
assert isinstance(parsed, dict), (
|
||||
f"Expected a JSON object (envelope), got {type(parsed)}"
|
||||
)
|
||||
_envelope_keys = {"command", "status", "exit_code", "data", "timing", "messages"}
|
||||
assert _envelope_keys.issubset(parsed.keys()), (
|
||||
f"Expected envelope keys {_envelope_keys}, got {set(parsed.keys())}"
|
||||
)
|
||||
|
||||
|
||||
@then('the json tree output should contain "{text}"')
|
||||
def step_tree_json_contains(context: Context, text: str) -> None:
|
||||
assert text in context.pe_tree_json, f"Expected '{text}' in tree JSON"
|
||||
|
||||
@@ -252,14 +252,18 @@ def step_langgraph_graph_with_max_retries(context: Any, retries: int) -> None:
|
||||
"I check langgraph should_retry with {status} validation and retry_count {count:d}"
|
||||
)
|
||||
def step_check_langgraph_should_retry(context: Any, status: str, count: int) -> None:
|
||||
"""Check should_retry decision."""
|
||||
"""Check should_retry decision.
|
||||
|
||||
Note: _should_retry is a conditional-edge function and is read-only from
|
||||
LangGraph's perspective. The state dict passed here is NOT mutated by
|
||||
the function (retry_count is incremented inside _validate instead).
|
||||
"""
|
||||
state: dict[str, Any] = {
|
||||
"validation_result": {"status": status},
|
||||
"retry_count": count,
|
||||
}
|
||||
decision = context.graph._should_retry(state)
|
||||
context.retry_decision = decision
|
||||
context.final_retry_count = state.get("retry_count", count)
|
||||
|
||||
|
||||
@then('the langgraph retry decision should be "{decision}"')
|
||||
@@ -283,6 +287,28 @@ def step_langgraph_validation_status(context: Any, status: str) -> None:
|
||||
assert validation.get("status") == status
|
||||
|
||||
|
||||
@then(
|
||||
"the langgraph validate result should include retry_count incremented from {before:d} to {after:d}"
|
||||
)
|
||||
def step_langgraph_validate_result_retry_count(
|
||||
context: Any, before: int, after: int
|
||||
) -> None:
|
||||
"""Verify _validate returns an incremented retry_count.
|
||||
|
||||
The _validate node is responsible for incrementing retry_count so that
|
||||
LangGraph persists the new value into the graph state. Conditional-edge
|
||||
functions such as _should_retry are read-only from LangGraph's perspective
|
||||
and must not mutate state.
|
||||
"""
|
||||
result = context.node_result
|
||||
assert "retry_count" in result, (
|
||||
"_validate must return retry_count in its result dict"
|
||||
)
|
||||
assert result["retry_count"] == after, (
|
||||
f"Expected retry_count {after} (incremented from {before}), got {result['retry_count']}"
|
||||
)
|
||||
|
||||
|
||||
@then('the langgraph validation message should contain "{text}"')
|
||||
def step_langgraph_validation_message_contains(context: Any, text: str) -> None:
|
||||
"""Verify validation message contains text."""
|
||||
|
||||
@@ -459,16 +459,32 @@ def step_have_state_with_retry_count(context: Any, count: int) -> None:
|
||||
"I check uncovered langgraph should_retry with FAIL validation and retry_count {count:d}"
|
||||
)
|
||||
def step_check_should_retry_uncovered(context: Any, count: int) -> None:
|
||||
"""Check should_retry and verify retry_count increment."""
|
||||
"""Check should_retry returns correct decision.
|
||||
|
||||
_should_retry is a conditional-edge function — LangGraph treats it as
|
||||
read-only. The retry counter is already incremented inside _validate
|
||||
(a node whose return dict IS merged into the state), so _handle_retry
|
||||
is a simple pass-through bridge node.
|
||||
"""
|
||||
decision = context.graph._should_retry(context.state)
|
||||
context.retry_decision = decision
|
||||
context.final_retry_count = context.state.get("retry_count")
|
||||
|
||||
|
||||
@then("the uncovered langgraph state retry_count should be incremented to {expected:d}")
|
||||
def step_uncovered_retry_count_incremented(context: Any, expected: int) -> None:
|
||||
"""Verify retry_count was incremented."""
|
||||
assert context.final_retry_count == expected
|
||||
@then(
|
||||
"the uncovered langgraph state retry_count should not be mutated and remain {expected:d}"
|
||||
)
|
||||
def step_uncovered_retry_count_not_mutated(context: Any, expected: int) -> None:
|
||||
"""Verify _should_retry did not mutate retry_count in state.
|
||||
|
||||
_should_retry is a conditional-edge function; LangGraph treats it as
|
||||
read-only. The retry counter is incremented inside _validate (a node)
|
||||
so that LangGraph persists the new value into the graph state.
|
||||
"""
|
||||
assert context.final_retry_count == expected, (
|
||||
f"_should_retry must not mutate state['retry_count']: "
|
||||
f"expected {expected}, got {context.final_retry_count}"
|
||||
)
|
||||
|
||||
|
||||
# Workflow retry scenario
|
||||
|
||||
@@ -184,7 +184,7 @@ def step_create_plan_with_description(context: Context, description: str) -> Non
|
||||
context.plan = _default_plan(description=description)
|
||||
|
||||
|
||||
@given("I create a plan in strategize phase")
|
||||
@given("I create a PlanModel in strategize phase")
|
||||
def step_create_plan_strategize_phase(context: Context) -> None:
|
||||
"""Create a plan in STRATEGIZE phase."""
|
||||
context.plan = _default_plan(
|
||||
@@ -204,7 +204,7 @@ def step_create_plan_strategize_with_action_state(context: Context) -> None:
|
||||
)
|
||||
|
||||
|
||||
@given("I create a plan in strategize phase with errored state")
|
||||
@given("I create a PlanModel in strategize phase with errored state")
|
||||
def step_create_plan_strategize_errored(context: Context) -> None:
|
||||
"""Create a plan in STRATEGIZE phase with ERRORED state."""
|
||||
context.plan = _default_plan(
|
||||
@@ -214,7 +214,7 @@ def step_create_plan_strategize_errored(context: Context) -> None:
|
||||
)
|
||||
|
||||
|
||||
@given("I create a plan in strategize phase with processing state")
|
||||
@given("I create a PlanModel in strategize phase with processing state")
|
||||
def step_create_plan_strategize_processing(context: Context) -> None:
|
||||
"""Create a plan in STRATEGIZE phase with PROCESSING state."""
|
||||
context.plan = _default_plan(
|
||||
@@ -268,7 +268,7 @@ def step_create_plan_action_available(context: Context) -> None:
|
||||
)
|
||||
|
||||
|
||||
@given("I create a plan in strategize phase with cancelled state")
|
||||
@given("I create a PlanModel in strategize phase with cancelled state")
|
||||
def step_create_plan_strategize_cancelled(context: Context) -> None:
|
||||
"""Create a plan in STRATEGIZE phase with CANCELLED processing state."""
|
||||
context.plan = _default_plan(
|
||||
|
||||
@@ -4,9 +4,9 @@ These steps target specific uncovered lines in
|
||||
cleveragents/infrastructure/plugins/loader.py:
|
||||
|
||||
- Lines 203-209: except block in load_from_entry_points when ep.load() fails
|
||||
- Lines 242, 244-246: validate_protocol fallback to issubclass on
|
||||
instantiation failure
|
||||
- Lines 247-248: validate_protocol when issubclass raises TypeError
|
||||
- validate_protocol: issubclass succeeds (class satisfies protocol structurally)
|
||||
- validate_protocol: issubclass raises TypeError for unverifiable protocol
|
||||
- validate_protocol: issubclass returns False (class missing required members)
|
||||
"""
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
@@ -72,13 +72,13 @@ def step_verify_warning_logged(context):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass succeeds (lines 242, 244-246)
|
||||
# validate_protocol: issubclass succeeds for class satisfying protocol
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SampleProtocol(Protocol):
|
||||
"""A simple runtime-checkable Protocol for testing issubclass fallback."""
|
||||
"""A simple runtime-checkable Protocol for testing issubclass check."""
|
||||
|
||||
def do_work(self) -> str: ...
|
||||
|
||||
@@ -105,7 +105,7 @@ def step_protocol_satisfied_by_subclass(context):
|
||||
|
||||
@when("I call validate_protocol with the non-instantiable class and protocol")
|
||||
def step_call_validate_protocol_subclass_fallback(context):
|
||||
"""Call validate_protocol; expect it to fall back to issubclass and succeed."""
|
||||
"""Call validate_protocol; issubclass succeeds and returns True."""
|
||||
context.validate_result = PluginLoader.validate_protocol(
|
||||
context.non_instantiable_class,
|
||||
context.target_protocol,
|
||||
@@ -114,18 +114,18 @@ def step_call_validate_protocol_subclass_fallback(context):
|
||||
|
||||
@then("validate_protocol should return True")
|
||||
def step_verify_validate_true(context):
|
||||
"""The fallback issubclass check should have returned True."""
|
||||
"""The issubclass check should have returned True."""
|
||||
assert context.validate_result is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass raises TypeError (lines 247-248)
|
||||
# validate_protocol: issubclass raises TypeError for unverifiable protocol
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("I have a class that cannot be instantiated without arguments")
|
||||
def step_class_cannot_instantiate(context):
|
||||
"""Create a class whose __init__ raises when called with no args."""
|
||||
"""Create a class whose __init__ requires mandatory arguments."""
|
||||
|
||||
class NeedsArgs:
|
||||
def __init__(self, required):
|
||||
@@ -141,10 +141,10 @@ def step_protocol_causes_typeerror(context):
|
||||
We need an object that:
|
||||
- Has a __name__ attribute (so the error message in validate_protocol works)
|
||||
- Causes issubclass() to raise TypeError when used as second arg
|
||||
- Also causes isinstance() to raise TypeError
|
||||
|
||||
A class with a metaclass that raises TypeError on __instancecheck__
|
||||
and __subclasscheck__ achieves this.
|
||||
A class with a metaclass that raises TypeError on __subclasscheck__
|
||||
achieves this. Since this protocol declares no members, the structural
|
||||
fallback cannot verify conformance and must raise ProtocolMismatchError.
|
||||
"""
|
||||
|
||||
class TypeErrorMeta(type):
|
||||
@@ -186,7 +186,7 @@ def step_verify_protocol_mismatch(context):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_protocol: instantiation fails, issubclass returns False
|
||||
# validate_protocol: issubclass returns False (class missing required members)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
AGENT_DEF_PATH = PROJECT_ROOT / ".opencode" / "agents" / "implementation-supervisor.md"
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Step definitions for PR compliance checklist in implementation pool supervisor.
|
||||
|
||||
This file uses parameterized @then decorators with unique step text that
|
||||
distinguishes pool-supervisor checks from the shared compliance checklist
|
||||
steps in pr_compliance_checklist_steps.py, preventing Behave AmbiguousStep
|
||||
errors when both feature files are run together.
|
||||
|
||||
Each validator is imported from the shared pr_compliance_checklist_steps module's
|
||||
validation logic (via the _verify module) to avoid code duplication while using
|
||||
unique step text prefixes ("Pool:") for disambiguation.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
AGENT_DEF_PATH = (
|
||||
PROJECT_ROOT / ".opencode" / "agents" / "implementation-pool-supervisor.md"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared validation helpers — identical logic to pr_compliance_checklist_steps.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_required_items = [
|
||||
"CHANGELOG.md",
|
||||
"CONTRIBUTORS.md",
|
||||
"ISSUES CLOSED",
|
||||
"CI passes",
|
||||
"BDD/Behave tests",
|
||||
"Epic reference",
|
||||
"forgejo-label-manager",
|
||||
"earliest open milestone",
|
||||
]
|
||||
|
||||
VALIDATORS: dict[str, Callable[[str], bool]] = {
|
||||
"includes checklist section": lambda c: "PR Compliance Checklist" in c,
|
||||
"is marked MANDATORY": lambda c: "MANDATORY" in c,
|
||||
"has CHANGELOG.md item": lambda c: "CHANGELOG.md" in c,
|
||||
"references Unreleased": lambda c: "[Unreleased]" in c,
|
||||
"has CONTRIBUTORS.md item": lambda c: "CONTRIBUTORS.md" in c,
|
||||
"instructs add or update": lambda c: "add or update" in c,
|
||||
"has commit footer item": lambda c: "Commit footer" in c,
|
||||
"specifies ISSUES CLOSED": lambda c: "ISSUES CLOSED" in c,
|
||||
"has CI passes item": lambda c: "CI passes" in c,
|
||||
"mentions quality gates": lambda c: "quality gates" in c,
|
||||
"has BDD tests item": lambda c: "BDD/Behave tests" in c,
|
||||
"instructs add or update features": lambda c: "added or updated" in c,
|
||||
"has Epic reference item": lambda c: "Epic reference" in c,
|
||||
"references parent Epic": lambda c: "parent Epic" in c,
|
||||
"has labels item": lambda c: "Labels" in c,
|
||||
"mentions forgejo-label-manager": lambda c: "forgejo-label-manager" in c,
|
||||
"has milestone item": lambda c: "Milestone" in c,
|
||||
"earliest open milestone": lambda c: "earliest open milestone" in c,
|
||||
"all 8 items present": lambda c: all(item in c for item in _required_items),
|
||||
}
|
||||
|
||||
|
||||
def _make_validator(key: str) -> Callable[[Any], None]:
|
||||
"""Factory that creates a typed Behave validator from a shared helper."""
|
||||
|
||||
def validator(context: Any) -> None:
|
||||
content = context.agent_def_content
|
||||
check_fn = VALIDATORS.get(key)
|
||||
assert check_fn(content), f"Pool supervisor agent definition failed: {key}"
|
||||
|
||||
return validator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unique @given and @when — scoped to the pool supervisor agent def only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the implementation-pool-supervisor.md agent definition exists")
|
||||
def step_agent_def_exists(context: Any) -> None:
|
||||
"""Verify the pool supervisor agent definition file exists."""
|
||||
assert AGENT_DEF_PATH.exists(), f"Agent definition not found at {AGENT_DEF_PATH}"
|
||||
context.agent_def_path = AGENT_DEF_PATH
|
||||
|
||||
|
||||
@when("I read the pool supervisor agent definition")
|
||||
def step_read_agent_def(context: Any) -> None:
|
||||
"""Read the pool supervisor agent definition."""
|
||||
context.agent_def_content = AGENT_DEF_PATH.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unique @then — prefixed with "Pool:" so they never conflict with the
|
||||
# shared pr_compliance_checklist_steps.py step definitions.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Scenario: Pool supervisor worker prompt includes the PR compliance checklist
|
||||
@then("Pool: worker prompt body includes the PR compliance checklist section")
|
||||
def pool_step_prompt_includes_checklist(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes the PR compliance checklist."""
|
||||
_make_validator("includes checklist section")(context)
|
||||
|
||||
|
||||
@then("Pool: the checklist is marked as MANDATORY")
|
||||
def pool_step_checklist_is_mandatory(context: Any) -> None:
|
||||
"""Verify the pool-supervisor checklist is marked as MANDATORY."""
|
||||
_make_validator("is marked MANDATORY")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 1 — CHANGELOG.md update required
|
||||
@then("Pool: worker prompt body includes a CHANGELOG.md checklist item")
|
||||
def pool_step_prompt_includes_changelog_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a CHANGELOG.md checklist item."""
|
||||
_make_validator("has CHANGELOG.md item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to add an entry under the Unreleased section")
|
||||
def pool_step_changelog_item_unreleased(context: Any) -> None:
|
||||
"""Verify the CHANGELOG.md item mentions the Unreleased section."""
|
||||
_make_validator("references Unreleased")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 2 — CONTRIBUTORS.md update required
|
||||
@then("Pool: worker prompt body includes a CONTRIBUTORS.md checklist item")
|
||||
def pool_step_prompt_includes_contributors_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a CONTRIBUTORS.md checklist item."""
|
||||
_make_validator("has CONTRIBUTORS.md item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to add or update their contribution entry")
|
||||
def pool_step_contributors_item_add_update(context: Any) -> None:
|
||||
"""Verify the CONTRIBUTORS.md item instructs workers to add or update."""
|
||||
_make_validator("instructs add or update")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 3 — commit footer required
|
||||
@then("Pool: worker prompt body includes a commit footer checklist item")
|
||||
def pool_step_prompt_includes_commit_footer_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a commit footer checklist item."""
|
||||
_make_validator("has commit footer item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item specifies the ISSUES CLOSED footer format")
|
||||
def pool_step_commit_footer_issues_closed(context: Any) -> None:
|
||||
"""Verify the commit footer item specifies the ISSUES CLOSED format."""
|
||||
_make_validator("specifies ISSUES CLOSED")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 4 — CI must pass before PR creation
|
||||
@then("Pool: worker prompt body includes a CI passes checklist item")
|
||||
def pool_step_prompt_includes_ci_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a CI passes checklist item."""
|
||||
_make_validator("has CI passes item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to verify all quality gates are green")
|
||||
def pool_step_ci_item_quality_gates(context: Any) -> None:
|
||||
"""Verify the CI item instructs workers to verify quality gates are green."""
|
||||
_make_validator("mentions quality gates")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 5 — BDD/Behave tests required
|
||||
@then("Pool: worker prompt body includes a BDD tests checklist item")
|
||||
def pool_step_prompt_includes_bdd_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a BDD/Behave tests checklist item."""
|
||||
_make_validator("has BDD tests item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to add or update Behave feature files")
|
||||
def pool_step_bdd_item_feature_files(context: Any) -> None:
|
||||
"""Verify the BDD item instructs workers to add or update feature files."""
|
||||
_make_validator("instructs add or update features")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 6 — Epic reference required in PR description
|
||||
@then("Pool: worker prompt body includes an Epic reference checklist item")
|
||||
def pool_step_prompt_includes_epic_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes an Epic reference checklist item."""
|
||||
_make_validator("has Epic reference item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to reference the parent Epic issue number")
|
||||
def pool_step_epic_item_parent_reference(context: Any) -> None:
|
||||
"""Verify the Epic item instructs workers to reference the parent Epic."""
|
||||
_make_validator("references parent Epic")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 7 — Labels must be applied
|
||||
@then("Pool: worker prompt body includes a labels checklist item")
|
||||
def pool_step_prompt_includes_labels_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a labels checklist item."""
|
||||
_make_validator("has labels item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to apply labels via forgejo-label-manager")
|
||||
def pool_step_labels_item_forgejo_label_manager(context: Any) -> None:
|
||||
"""Verify the labels item instructs workers to use forgejo-label-manager."""
|
||||
_make_validator("mentions forgejo-label-manager")(context)
|
||||
|
||||
|
||||
# Scenario: Checklist item 8 — Milestone must be assigned
|
||||
@then("Pool: worker prompt body includes a milestone checklist item")
|
||||
def pool_step_prompt_includes_milestone_item(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body includes a milestone checklist item."""
|
||||
_make_validator("has milestone item")(context)
|
||||
|
||||
|
||||
@then("Pool: the item instructs workers to assign the earliest open milestone")
|
||||
def pool_step_milestone_item_earliest(context: Any) -> None:
|
||||
"""Verify the milestone item instructs workers to assign the earliest open milestone."""
|
||||
_make_validator("earliest open milestone")(context)
|
||||
|
||||
|
||||
# Scenario: All 8 checklist items are present in the worker prompt
|
||||
@then("Pool: worker prompt body contains all 8 mandatory checklist items")
|
||||
def pool_step_prompt_contains_all_8_items(context: Any) -> None:
|
||||
"""Verify the pool-supervisor worker prompt body contains all 8 mandatory checklist items."""
|
||||
_make_validator("all 8 items present")(context)
|
||||
@@ -183,3 +183,54 @@ def step_apply_less_or_equal(context: Any) -> None:
|
||||
exec_tokens = context.phase_result["phases"]["execute"]["total_tokens"]
|
||||
apply_tokens = context.phase_result["phases"]["apply"]["total_tokens"]
|
||||
assert apply_tokens <= exec_tokens
|
||||
|
||||
|
||||
@given("a phase analysis policy with opencode exclude paths")
|
||||
def step_policy_opencode_exclude(context: Any) -> None:
|
||||
"""Policy that excludes .opencode/** paths using relative globs."""
|
||||
context.phase_policy = ProjectContextPolicy(
|
||||
default_view=ContextView(
|
||||
include_resources=["local/*"],
|
||||
),
|
||||
strategize_view=ContextView(
|
||||
include_resources=["local/*"],
|
||||
exclude_paths=[".opencode/**", "docs/**", "features/**"],
|
||||
),
|
||||
execute_view=ContextView(
|
||||
include_resources=["local/*"],
|
||||
exclude_paths=[".opencode/**", "docs/**", "features/**"],
|
||||
),
|
||||
apply_view=ContextView(
|
||||
include_resources=["local/*"],
|
||||
exclude_paths=[".opencode/**", "docs/**", "features/**"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@given("an absolute path fragment for phase analysis")
|
||||
def step_absolute_path_fragment(context: Any) -> None:
|
||||
"""Fragment with an absolute path that should be excluded by relative globs."""
|
||||
context.phase_fragments = [
|
||||
TieredFragment(
|
||||
fragment_id="abs-skill",
|
||||
content="skill content",
|
||||
tier=ContextTier.HOT,
|
||||
resource_id="local/repo-a",
|
||||
project_name="local/ctx-app",
|
||||
token_count=50,
|
||||
metadata={
|
||||
"path": "/app/.opencode/skills/SKILL.md",
|
||||
"byte_size": 1000,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@then("strategize phase should exclude the absolute path fragment")
|
||||
def step_strat_excludes_absolute(context: Any) -> None:
|
||||
"""Verify that the absolute path fragment is excluded by relative glob patterns."""
|
||||
strat = context.phase_result["phases"]["strategize"]
|
||||
assert strat["fragment_count"] == 0, (
|
||||
f"Expected 0 fragments (absolute path should be excluded by relative glob), "
|
||||
f"got {strat['fragment_count']}"
|
||||
)
|
||||
|
||||
@@ -492,7 +492,7 @@ def step_pr_remove_link_by_id(context: Any, link_id: str) -> None:
|
||||
context.pr_remove_result = context.pr_link_repo.remove_link(link_id)
|
||||
|
||||
|
||||
@then("the remove result should be False")
|
||||
@then("the project repo remove result should be False")
|
||||
def step_pr_remove_false(context: Any) -> None:
|
||||
assert context.pr_remove_result is False
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Step definitions for PyYAML security dependency verification (#9055).
|
||||
|
||||
Note: the shared steps for reading pyproject.toml and verifying importability
|
||||
are defined in ``tdd_a2a_sdk_dependency_steps.py`` to avoid AmbiguousStep
|
||||
conflicts. This file contains only the PyYAML-specific step definitions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from behave import then, when
|
||||
from behave.runner import Context
|
||||
from packaging.version import parse as parse_version
|
||||
|
||||
|
||||
@then('the dependency list should include a package matching "{pattern}"')
|
||||
def step_dependency_matches_pattern(context: Context, pattern: str) -> None:
|
||||
"""Assert that a dependency in the project dependencies matches the given regex pattern."""
|
||||
deps: list[str] = context.project_dependencies
|
||||
found = any(re.search(pattern, dep) for dep in deps)
|
||||
assert found, (
|
||||
f"No dependency matching '{pattern}' found in "
|
||||
f"[project.dependencies]. Current deps: {deps}"
|
||||
)
|
||||
|
||||
|
||||
@when("I read the PyYAML dependency specification from pyproject.toml")
|
||||
def step_read_pyyaml_dependency(context: Context) -> None:
|
||||
"""Read and parse the PyYAML dependency from pyproject.toml."""
|
||||
content = context.pyproject_path.read_bytes()
|
||||
data: dict[str, Any] = _load_toml(content)
|
||||
deps: list[str] = data.get("project", {}).get("dependencies", [])
|
||||
|
||||
# Find the pyyaml dependency entry
|
||||
pyyaml_dep = None
|
||||
for dep in deps:
|
||||
if dep.strip().startswith("pyyaml"):
|
||||
pyyaml_dep = dep.strip()
|
||||
break
|
||||
|
||||
assert pyyaml_dep is not None, (
|
||||
f"PyYAML dependency not found in [project.dependencies]. Current deps: {deps}"
|
||||
)
|
||||
|
||||
# Parse the minimum version from the constraint
|
||||
# Examples: "pyyaml>=6.0.3", "pyyaml>=6.0", "pyyaml==6.0.2"
|
||||
match = re.search(r"(?:>=|==|~>)([\d]+(?:\.[\d]+)*(?:\.\w+)*)", pyyaml_dep)
|
||||
assert match is not None, (
|
||||
f"Could not parse version from PyYAML dependency: {pyyaml_dep}"
|
||||
)
|
||||
|
||||
context.pyyaml_version_spec = pyyaml_dep
|
||||
context.pyyaml_min_version = match.group(1)
|
||||
|
||||
|
||||
@then('the minimum version should be at least "{expected}"')
|
||||
def step_pyyaml_minimum_version(context: Context, expected: str) -> None:
|
||||
"""Assert that the PyYAML minimum version is >= expected version."""
|
||||
min_version = context.pyyaml_min_version
|
||||
assert parse_version(min_version) >= parse_version(expected), (
|
||||
f"PyYAML minimum version {min_version} is less than required {expected}. "
|
||||
f"Dependency spec: {context.pyyaml_version_spec}"
|
||||
)
|
||||
|
||||
|
||||
def _flatten_toml_dict(d: dict[Any, Any]) -> dict[str, Any]:
|
||||
"""Recursively convert a tomlkit dict to a plain Python dict.
|
||||
|
||||
tomlkit wraps every value in a proxy object that behaves like the
|
||||
underlying Python type but is not identical to it. This helper
|
||||
strips those wrappers so the result is a plain ``dict[str, Any]``
|
||||
compatible with ``tomllib`` output.
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in d.items():
|
||||
if isinstance(value, dict):
|
||||
result[key] = _flatten_toml_dict(value)
|
||||
else:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _toml_to_python(data: object) -> dict[str, Any]:
|
||||
"""Convert a tomlkit document to a plain Python dict.
|
||||
|
||||
Delegates recursive flattening to the module-level
|
||||
``_flatten_toml_dict`` helper so the logic stays testable in
|
||||
isolation and ruff does not flag nested function definitions.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
return _flatten_toml_dict(data)
|
||||
raise TypeError(f"Expected dict-like TOML table, got {type(data).__name__}")
|
||||
|
||||
|
||||
def _load_toml(content: bytes) -> dict[str, Any]:
|
||||
"""Load a TOML file from bytes. Uses tomllib if available, else fallback."""
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
return tomllib.loads(content.decode("utf-8"))
|
||||
except ImportError: # pragma: no cover — Python <3.11
|
||||
# Fallback for older Python versions without tomllib
|
||||
import tomlkit
|
||||
|
||||
parsed_doc = tomlkit.parse(content.decode("utf-8"))
|
||||
return _toml_to_python(parsed_doc)
|
||||
@@ -185,7 +185,7 @@ def step_settings_with_overrides(context: Any, value: int) -> None:
|
||||
os.environ.pop("CLEVERAGENTS_RETRY_SERVICE_OVERRIDES", None)
|
||||
|
||||
|
||||
@when("I create a ServiceRetryWiring from those Settings")
|
||||
@when("I create a ServiceRetryWiring from those retry Settings")
|
||||
def step_create_wiring_from_settings(context: Any) -> None:
|
||||
from cleveragents.application.services.service_retry_wiring import (
|
||||
ServiceRetryWiring,
|
||||
|
||||
@@ -27,6 +27,7 @@ from unittest.mock import MagicMock, patch
|
||||
from behave import given, then, when
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.application.services.session_workflow import TellResult
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.core.exceptions import DatabaseError
|
||||
from cleveragents.domain.models.core.cost_budget import SessionCostBudget
|
||||
@@ -101,15 +102,13 @@ def _mock_service() -> MagicMock:
|
||||
|
||||
|
||||
def _patch_service(context, svc):
|
||||
"""Patch _get_session_service to return *svc* deterministically.
|
||||
"""Patch ``_get_session_service`` to return *svc* deterministically.
|
||||
|
||||
Patching the function directly (rather than the module-level ``_service``
|
||||
attribute) prevents a race condition in parallel Behave workers: if a
|
||||
concurrent worker's cleanup resets ``_service`` to ``None`` while this
|
||||
scenario's CLI command is executing, ``_get_session_service()`` would fall
|
||||
through to the real DI container and raise, producing a spurious exit
|
||||
code 1. Patching the function itself short-circuits the cache check
|
||||
entirely and is immune to cross-worker ``_service`` mutations.
|
||||
Patches the function rather than mutating the module-level ``_service``
|
||||
singleton to prevent a race condition in parallel Behave workers.
|
||||
Mutating ``_service`` could cause concurrent cleanup in one worker to
|
||||
reset it to ``None`` while another worker was still using it, leading
|
||||
to intermittent test failures.
|
||||
"""
|
||||
patcher = patch(
|
||||
"cleveragents.cli.commands.session._get_session_service",
|
||||
@@ -119,6 +118,33 @@ def _patch_service(context, svc):
|
||||
context._scvbst_cleanups.append(patcher.stop)
|
||||
|
||||
|
||||
def _patch_workflow(context, wf):
|
||||
"""Patch _build_session_workflow to return *wf* deterministically."""
|
||||
patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
return_value=wf,
|
||||
)
|
||||
patcher.start()
|
||||
context._scvbst_cleanups.append(patcher.stop)
|
||||
|
||||
|
||||
def _make_tell_result(
|
||||
session_id: str = _ULID1,
|
||||
prompt: str = "Hello world",
|
||||
response: str = "Acknowledged: Hello world",
|
||||
) -> TellResult:
|
||||
return TellResult(
|
||||
session_id=session_id,
|
||||
user_message=prompt,
|
||||
assistant_message=response,
|
||||
input_tokens=5,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -601,7 +627,14 @@ def step_invoke_import_db_error(context):
|
||||
def step_tell_service(context):
|
||||
svc = _mock_service()
|
||||
svc.append_message.return_value = None
|
||||
svc.get_messages.return_value = []
|
||||
_patch_service(context, svc)
|
||||
# Patch workflow so tell uses a canned response without LLM/DB.
|
||||
wf = MagicMock()
|
||||
wf.tell.return_value = _make_tell_result()
|
||||
wf.tell_stream.return_value = iter(["Acknowledged: Hello stream"])
|
||||
_patch_workflow(context, wf)
|
||||
context._scvbst_mock_wf = wf
|
||||
|
||||
|
||||
@when("session coverage boost I invoke the tell command without stream")
|
||||
@@ -622,6 +655,11 @@ def step_invoke_tell_stream(context):
|
||||
|
||||
@when("session coverage boost I invoke the tell command with actor override")
|
||||
def step_invoke_tell_actor(context):
|
||||
# Override the mock workflow response to include actor name
|
||||
context._scvbst_mock_wf.tell.return_value = _make_tell_result(
|
||||
prompt="Plan a feature",
|
||||
response="[openai/gpt-4] Acknowledged: Plan a feature",
|
||||
)
|
||||
context.result = _runner.invoke(
|
||||
session_app,
|
||||
[
|
||||
@@ -647,6 +685,11 @@ def step_tell_not_found(context):
|
||||
svc = _mock_service()
|
||||
svc.append_message.side_effect = SessionNotFoundError("no session")
|
||||
_patch_service(context, svc)
|
||||
# Workflow raises SessionNotFoundError so the CLI handler catches it.
|
||||
wf = MagicMock()
|
||||
wf.tell.side_effect = SessionNotFoundError("no session")
|
||||
wf.tell_stream.side_effect = SessionNotFoundError("no session")
|
||||
_patch_workflow(context, wf)
|
||||
|
||||
|
||||
@given("session coverage boost a mock service that raises DatabaseError on append")
|
||||
@@ -654,6 +697,11 @@ def step_tell_db_error(context):
|
||||
svc = _mock_service()
|
||||
svc.append_message.side_effect = DatabaseError("tell db fail")
|
||||
_patch_service(context, svc)
|
||||
# Workflow raises DatabaseError so the CLI handler catches it.
|
||||
wf = MagicMock()
|
||||
wf.tell.side_effect = DatabaseError("tell db fail")
|
||||
wf.tell_stream.side_effect = DatabaseError("tell db fail")
|
||||
_patch_workflow(context, wf)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,14 +8,14 @@ import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
from ulid import ULID
|
||||
|
||||
from cleveragents.cli.commands import session as session_mod
|
||||
from cleveragents.application.services.session_workflow import TellResult
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.domain.models.core.session import (
|
||||
MessageRole,
|
||||
@@ -98,12 +98,43 @@ def step_session_cli_runner(context: Context) -> None:
|
||||
# Default: create returns a session
|
||||
default_session = _make_session(session_id=_SESSION_ID)
|
||||
context.mock_service.create.return_value = default_session
|
||||
# get_messages returns empty list by default (used by SessionWorkflow)
|
||||
context.mock_service.get_messages.return_value = []
|
||||
|
||||
# Patch the module-level service
|
||||
session_mod._service = context.mock_service
|
||||
# Patch _get_session_service so tests are safe for parallel workers
|
||||
# and do not mutate the module-level _service cache (M9).
|
||||
svc_patcher = patch(
|
||||
"cleveragents.cli.commands.session._get_session_service",
|
||||
return_value=context.mock_service,
|
||||
)
|
||||
svc_patcher.start()
|
||||
|
||||
# Patch _build_session_workflow to return a controllable mock workflow.
|
||||
# This lets tell-specific tests configure exact return values without
|
||||
# needing a real LLM or provider registry in the test environment.
|
||||
context.mock_workflow = MagicMock()
|
||||
_default_tell_result = TellResult(
|
||||
session_id=_SESSION_ID,
|
||||
user_message="",
|
||||
assistant_message="Acknowledged",
|
||||
input_tokens=10,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
context.mock_workflow.tell.return_value = _default_tell_result
|
||||
context.mock_workflow.tell_stream.return_value = iter(["Acknowledged"])
|
||||
|
||||
workflow_patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
return_value=context.mock_workflow,
|
||||
)
|
||||
workflow_patcher.start()
|
||||
|
||||
def cleanup() -> None:
|
||||
session_mod._service = None
|
||||
svc_patcher.stop()
|
||||
workflow_patcher.stop()
|
||||
_cleanup(context)
|
||||
|
||||
context.add_cleanup(cleanup)
|
||||
@@ -306,10 +337,17 @@ def step_capture_first_ulid(context: Context) -> None:
|
||||
def step_tell_with_stored_ulid(context: Context, prompt: str) -> None:
|
||||
"""Run session tell using the full ULID stored from session list."""
|
||||
session_id = context.full_session_id
|
||||
context.mock_service.append_message.side_effect = [
|
||||
_make_message(MessageRole.USER, prompt, 0),
|
||||
_make_message(MessageRole.ASSISTANT, f"Acknowledged: {prompt}", 1),
|
||||
]
|
||||
# Configure mock workflow for this prompt.
|
||||
context.mock_workflow.tell.return_value = TellResult(
|
||||
session_id=session_id,
|
||||
user_message=prompt,
|
||||
assistant_message=f"Acknowledged: {prompt}",
|
||||
input_tokens=len(prompt) // 4,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
context.result = context.runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", session_id, prompt],
|
||||
@@ -563,21 +601,37 @@ def step_import_succeeds(context: Context) -> None:
|
||||
|
||||
@given("there is a mocked session for tell")
|
||||
def step_session_for_tell(context: Context) -> None:
|
||||
session = _make_session(session_id=_SESSION_ID)
|
||||
# Session with actor so tell can proceed without --actor flag.
|
||||
session = _make_session(session_id=_SESSION_ID, actor_name="openai/gpt-4")
|
||||
context.mock_service.get.return_value = session
|
||||
context.mock_service.append_message.side_effect = [
|
||||
_make_message(MessageRole.USER, "Hello, world", 0),
|
||||
_make_message(MessageRole.ASSISTANT, "Acknowledged: Hello, world", 1),
|
||||
]
|
||||
context.mock_service.get_messages.return_value = []
|
||||
context.session_id = _SESSION_ID
|
||||
# Configure the mock workflow to return a canned TellResult.
|
||||
context.mock_workflow.tell.return_value = TellResult(
|
||||
session_id=_SESSION_ID,
|
||||
user_message="Hello, world",
|
||||
assistant_message="Acknowledged: Hello, world",
|
||||
input_tokens=10,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
|
||||
|
||||
@when('I run session CLI tell with a prompt "{prompt}"')
|
||||
def step_tell_prompt(context: Context, prompt: str) -> None:
|
||||
context.mock_service.append_message.side_effect = [
|
||||
_make_message(MessageRole.USER, prompt, 0),
|
||||
_make_message(MessageRole.ASSISTANT, f"Acknowledged: {prompt}", 1),
|
||||
]
|
||||
# Update the mock workflow result to reflect the current prompt.
|
||||
context.mock_workflow.tell.return_value = TellResult(
|
||||
session_id=context.session_id,
|
||||
user_message=prompt,
|
||||
assistant_message=f"Acknowledged: {prompt}",
|
||||
input_tokens=len(prompt) // 4,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
context.result = context.runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", context.session_id, prompt],
|
||||
@@ -586,14 +640,17 @@ def step_tell_prompt(context: Context, prompt: str) -> None:
|
||||
|
||||
@when('I run session CLI tell with --actor "{actor}" and prompt "{prompt}"')
|
||||
def step_tell_with_actor(context: Context, actor: str, prompt: str) -> None:
|
||||
context.mock_service.append_message.side_effect = [
|
||||
_make_message(MessageRole.USER, prompt, 0),
|
||||
_make_message(
|
||||
MessageRole.ASSISTANT,
|
||||
f"[{actor}] Acknowledged: {prompt}",
|
||||
1,
|
||||
),
|
||||
]
|
||||
# Update the mock workflow result to reflect actor and prompt.
|
||||
context.mock_workflow.tell.return_value = TellResult(
|
||||
session_id=context.session_id,
|
||||
user_message=prompt,
|
||||
assistant_message=f"[{actor}] Acknowledged: {prompt}",
|
||||
input_tokens=len(prompt) // 4,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
context.result = context.runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", context.session_id, "--actor", actor, prompt],
|
||||
@@ -602,9 +659,8 @@ def step_tell_with_actor(context: Context, actor: str, prompt: str) -> None:
|
||||
|
||||
@when("I run session CLI tell to a non-existent session")
|
||||
def step_tell_nonexistent(context: Context) -> None:
|
||||
context.mock_service.append_message.side_effect = SessionNotFoundError(
|
||||
"Session not found"
|
||||
)
|
||||
# Configure the workflow to raise SessionNotFoundError.
|
||||
context.mock_workflow.tell.side_effect = SessionNotFoundError("Session not found")
|
||||
context.result = context.runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", "NONEXISTENT", "Hello"],
|
||||
|
||||
@@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch
|
||||
from behave import given, then, when
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.application.services.session_workflow import TellResult
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.domain.models.core.session import (
|
||||
MessageRole,
|
||||
@@ -358,10 +359,28 @@ def step_export_output_contains(context, text):
|
||||
@given("session cli branch a mock session service for tell")
|
||||
def step_service_for_tell(context):
|
||||
svc = _mock_service()
|
||||
# append_message doesn't need to return anything meaningful for the
|
||||
# code path under test — the assistant content is computed inline.
|
||||
svc.append_message.return_value = None
|
||||
svc.get_messages.return_value = []
|
||||
_patch_service(context, svc)
|
||||
# Patch _build_session_workflow to return a controllable mock.
|
||||
mock_wf = MagicMock()
|
||||
mock_wf.tell_stream.return_value = iter(["Acknowledged: Hello world"])
|
||||
mock_wf.tell.return_value = TellResult(
|
||||
session_id=_ULID,
|
||||
user_message="Hello world",
|
||||
assistant_message="Acknowledged: Hello world",
|
||||
input_tokens=5,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
wf_patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
return_value=mock_wf,
|
||||
)
|
||||
wf_patcher.start()
|
||||
context._cleanup_handlers.append(wf_patcher.stop)
|
||||
|
||||
|
||||
@when("session cli branch I invoke the tell command with --stream")
|
||||
@@ -374,7 +393,8 @@ def step_invoke_tell_stream(context):
|
||||
|
||||
@then("session cli branch the streamed output contains the assistant response")
|
||||
def step_streamed_output(context):
|
||||
# The assistant content for no actor is: "Acknowledged: Hello world"
|
||||
# After the stub LLM replacement the assistant response comes from the
|
||||
# mock workflow's tell_stream. Check that ANY assistant output arrived.
|
||||
assert "Acknowledged" in context.result.output, (
|
||||
f"Expected 'Acknowledged' in output. Got:\n{context.result.output}"
|
||||
)
|
||||
|
||||
@@ -447,7 +447,7 @@ def session_model_check_export_messages_count(context: Context) -> None:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I get the session CLI dict")
|
||||
@when("I get the session model CLI dict")
|
||||
def session_model_get_cli_dict(context: Context) -> None:
|
||||
"""Get the CLI dict for the session."""
|
||||
context.session_cli_dict = context.session_model.as_cli_dict()
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
"""Step definitions for session tell real LLM invocation tests (issue #5784).
|
||||
|
||||
Tests verify that ``agents session tell`` routes through
|
||||
:class:`~cleveragents.application.services.session_workflow.SessionWorkflow`
|
||||
with a real (mock) :class:`LLMCaller`, persists the response, and records
|
||||
token usage.
|
||||
|
||||
All LLM interactions are handled via a mock ``LLMCaller`` so no real API
|
||||
keys or network access is required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.application.services.session_workflow import SessionWorkflow
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.domain.models.core.session import (
|
||||
MessageRole,
|
||||
Session,
|
||||
SessionMessage,
|
||||
SessionService,
|
||||
SessionTokenUsage,
|
||||
)
|
||||
|
||||
_ULID = "01KJ1N8YDN05N29P5RZV4SPDVZ"
|
||||
_STUB_RESPONSE = "Here is what I can help you with."
|
||||
|
||||
|
||||
class _StubResp:
|
||||
"""Minimal LLM response stub for _StubLLM.invoke()."""
|
||||
|
||||
def __init__(self, content: str) -> None:
|
||||
self.content = content
|
||||
self.tool_calls: list[Any] = []
|
||||
self.response_metadata: dict[str, Any] = {
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5}
|
||||
}
|
||||
|
||||
|
||||
class _StubChunk:
|
||||
"""Minimal streaming chunk for _StubLLM.stream()."""
|
||||
|
||||
def __init__(self, content: str) -> None:
|
||||
self.content = content
|
||||
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
class _StubLLM:
|
||||
"""Minimal LangChain-compatible LLM stub for LangChainSessionCaller tests."""
|
||||
|
||||
def __init__(self, response: str = _STUB_RESPONSE) -> None:
|
||||
self._response = response
|
||||
self.invoked_messages: list[Any] = []
|
||||
|
||||
def invoke(self, messages: Any, **kwargs: Any) -> _StubResp:
|
||||
self.invoked_messages = list(messages)
|
||||
return _StubResp(self._response)
|
||||
|
||||
def stream(self, messages: Any, **kwargs: Any) -> Iterator[_StubChunk]:
|
||||
self.invoked_messages = list(messages)
|
||||
words = self._response.split()
|
||||
for i, word in enumerate(words):
|
||||
# Reconstruct spaces between words so the concatenated
|
||||
# output matches the original response string verbatim.
|
||||
token = word if i == 0 else " " + word
|
||||
yield _StubChunk(token)
|
||||
|
||||
|
||||
def _make_session(
|
||||
session_id: str = _ULID,
|
||||
actor_name: str | None = None,
|
||||
) -> Session:
|
||||
return Session(
|
||||
session_id=session_id,
|
||||
actor_name=actor_name,
|
||||
namespace="local",
|
||||
messages=[],
|
||||
token_usage=SessionTokenUsage(),
|
||||
)
|
||||
|
||||
|
||||
def _make_mock_service(session: Session) -> MagicMock:
|
||||
"""Build a MagicMock SessionService pre-configured for session tell."""
|
||||
svc = MagicMock(spec=SessionService)
|
||||
svc.get.return_value = session
|
||||
svc.get_messages.return_value = []
|
||||
svc.append_message.return_value = MagicMock(spec=SessionMessage)
|
||||
svc.update_token_usage.return_value = None
|
||||
return svc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a session tell LLM mock environment")
|
||||
def step_setup_llm_mock_env(context: Context) -> None:
|
||||
"""Set up runner, stub LLM, and cleanup handlers."""
|
||||
context.runner = runner
|
||||
context.stub_llm = _StubLLM()
|
||||
context.captured_actor_name: str | None = None
|
||||
context._cleanup_handlers: list[Any] = []
|
||||
|
||||
def cleanup() -> None:
|
||||
for stop in reversed(context._cleanup_handlers):
|
||||
with contextlib.suppress(Exception):
|
||||
stop()
|
||||
|
||||
context.add_cleanup(cleanup)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a session with actor "{actor}" exists')
|
||||
def step_session_with_actor(context: Context, actor: str) -> None:
|
||||
session = _make_session(actor_name=actor)
|
||||
context.session = session
|
||||
context.svc = _make_mock_service(session)
|
||||
# Patch _get_session_service instead of module-level _service so tests
|
||||
# are resilient to changes in the internal caching strategy (m4).
|
||||
svc_patcher = patch(
|
||||
"cleveragents.cli.commands.session._get_session_service",
|
||||
return_value=context.svc,
|
||||
)
|
||||
svc_patcher.start()
|
||||
context._cleanup_handlers.append(svc_patcher.stop)
|
||||
|
||||
|
||||
@given("a session with no actor exists")
|
||||
def step_session_no_actor(context: Context) -> None:
|
||||
session = _make_session(actor_name=None)
|
||||
context.session = session
|
||||
context.svc = _make_mock_service(session)
|
||||
svc_patcher = patch(
|
||||
"cleveragents.cli.commands.session._get_session_service",
|
||||
return_value=context.svc,
|
||||
)
|
||||
svc_patcher.start()
|
||||
context._cleanup_handlers.append(svc_patcher.stop)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I invoke session tell with prompt "{prompt}"')
|
||||
def step_invoke_tell(context: Context, prompt: str) -> None:
|
||||
"""Invoke tell routing through _facade_dispatch.
|
||||
|
||||
The non-streaming CLI path calls :func:`_facade_dispatch`, which in
|
||||
production delegates to ``A2aLocalFacade``. The test intercepts
|
||||
``_facade_dispatch`` with a thin adapter that runs the real
|
||||
``SessionWorkflow.tell()`` (backed by the test's stub LLM) and
|
||||
returns a dict matching the facade's response envelope. This
|
||||
keeps the workflow assertions (append_message, token usage) working
|
||||
while also testing the CLI's facade-routing code path.
|
||||
"""
|
||||
context.prompt = prompt
|
||||
context.invoked_actor_name = None
|
||||
stub_llm = context.stub_llm
|
||||
svc = context.svc
|
||||
|
||||
def _llm_factory(actor: str) -> Any:
|
||||
context.invoked_actor_name = actor
|
||||
return stub_llm
|
||||
|
||||
wf = SessionWorkflow(session_service=svc, llm_factory=_llm_factory)
|
||||
|
||||
def _mock_dispatch(operation: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
if operation == "message/send":
|
||||
result = wf.tell(
|
||||
session_id=params["session_id"],
|
||||
prompt=params["message"],
|
||||
actor_override=params.get("actor"),
|
||||
)
|
||||
return {
|
||||
"session_id": result.session_id,
|
||||
"assistant_message": result.assistant_message,
|
||||
"usage": result.usage_dict(),
|
||||
}
|
||||
raise RuntimeError(f"Unknown operation: {operation}")
|
||||
|
||||
dispatch_patcher = patch(
|
||||
"cleveragents.cli.commands.session._facade_dispatch",
|
||||
side_effect=_mock_dispatch,
|
||||
)
|
||||
dispatch_patcher.start()
|
||||
context._cleanup_handlers.append(dispatch_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, prompt],
|
||||
)
|
||||
|
||||
|
||||
@when('I invoke session tell with --stream and prompt "{prompt}"')
|
||||
def step_invoke_tell_stream(context: Context, prompt: str) -> None:
|
||||
"""Invoke tell --stream with a real SessionWorkflow backed by _StubLLM."""
|
||||
context.prompt = prompt
|
||||
stub_llm = context.stub_llm
|
||||
svc = context.svc
|
||||
|
||||
def _workflow_factory() -> SessionWorkflow:
|
||||
return SessionWorkflow(
|
||||
session_service=svc,
|
||||
llm_factory=lambda actor: stub_llm,
|
||||
)
|
||||
|
||||
wf_patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
side_effect=_workflow_factory,
|
||||
)
|
||||
wf_patcher.start()
|
||||
context._cleanup_handlers.append(wf_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, "--stream", prompt],
|
||||
)
|
||||
|
||||
|
||||
@when('I invoke session tell with --actor "{actor}" and prompt "{prompt}"')
|
||||
def step_invoke_tell_with_actor(context: Context, actor: str, prompt: str) -> None:
|
||||
"""Invoke tell with --actor override, routing through _facade_dispatch."""
|
||||
context.prompt = prompt
|
||||
context.override_actor = actor
|
||||
stub_llm = context.stub_llm
|
||||
svc = context.svc
|
||||
|
||||
def _llm_factory(actor_name: str) -> Any:
|
||||
context.invoked_actor_name = actor_name
|
||||
return stub_llm
|
||||
|
||||
wf = SessionWorkflow(session_service=svc, llm_factory=_llm_factory)
|
||||
|
||||
def _mock_dispatch(operation: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
if operation == "message/send":
|
||||
result = wf.tell(
|
||||
session_id=params["session_id"],
|
||||
prompt=params["message"],
|
||||
actor_override=params.get("actor"),
|
||||
)
|
||||
return {
|
||||
"session_id": result.session_id,
|
||||
"assistant_message": result.assistant_message,
|
||||
"usage": result.usage_dict(),
|
||||
}
|
||||
raise RuntimeError(f"Unknown operation: {operation}")
|
||||
|
||||
dispatch_patcher = patch(
|
||||
"cleveragents.cli.commands.session._facade_dispatch",
|
||||
side_effect=_mock_dispatch,
|
||||
)
|
||||
dispatch_patcher.start()
|
||||
context._cleanup_handlers.append(dispatch_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, "--actor", actor, prompt],
|
||||
)
|
||||
|
||||
|
||||
@when('I invoke session tell with --format json and prompt "{prompt}"')
|
||||
def step_invoke_tell_json(context: Context, prompt: str) -> None:
|
||||
"""Invoke tell --format json, routing through _facade_dispatch."""
|
||||
context.prompt = prompt
|
||||
stub_llm = context.stub_llm
|
||||
svc = context.svc
|
||||
|
||||
wf = SessionWorkflow(
|
||||
session_service=svc,
|
||||
llm_factory=lambda actor: stub_llm,
|
||||
)
|
||||
|
||||
def _mock_dispatch(operation: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
if operation == "message/send":
|
||||
result = wf.tell(
|
||||
session_id=params["session_id"],
|
||||
prompt=params["message"],
|
||||
actor_override=params.get("actor"),
|
||||
)
|
||||
return {
|
||||
"session_id": result.session_id,
|
||||
"assistant_message": result.assistant_message,
|
||||
"usage": result.usage_dict(),
|
||||
}
|
||||
raise RuntimeError(f"Unknown operation: {operation}")
|
||||
|
||||
dispatch_patcher = patch(
|
||||
"cleveragents.cli.commands.session._facade_dispatch",
|
||||
side_effect=_mock_dispatch,
|
||||
)
|
||||
dispatch_patcher.start()
|
||||
context._cleanup_handlers.append(dispatch_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, "--format", "json", prompt],
|
||||
)
|
||||
|
||||
def _mock_dispatch(operation: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
if operation == "message/send":
|
||||
result = wf.tell(
|
||||
session_id=params["session_id"],
|
||||
prompt=params["message"],
|
||||
actor_override=params.get("actor"),
|
||||
)
|
||||
return {
|
||||
"session_id": result.session_id,
|
||||
"assistant_message": result.assistant_message,
|
||||
"usage": result.usage_dict(),
|
||||
}
|
||||
raise RuntimeError(f"Unknown operation: {operation}")
|
||||
|
||||
dispatch_patcher = patch(
|
||||
"cleveragents.cli.commands.session._facade_dispatch",
|
||||
side_effect=_mock_dispatch,
|
||||
)
|
||||
dispatch_patcher.start()
|
||||
context._cleanup_handlers.append(dispatch_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, "--format", "json", prompt],
|
||||
)
|
||||
|
||||
|
||||
@when('I invoke session tell with --stream --format json and prompt "{prompt}"')
|
||||
def step_invoke_tell_stream_json(context: Context, prompt: str) -> None:
|
||||
"""Invoke tell --stream --format json with a real SessionWorkflow backed by _StubLLM."""
|
||||
context.prompt = prompt
|
||||
stub_llm = context.stub_llm
|
||||
svc = context.svc
|
||||
|
||||
def _workflow_factory() -> SessionWorkflow:
|
||||
return SessionWorkflow(
|
||||
session_service=svc,
|
||||
llm_factory=lambda actor: stub_llm,
|
||||
)
|
||||
|
||||
wf_patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
side_effect=_workflow_factory,
|
||||
)
|
||||
wf_patcher.start()
|
||||
context._cleanup_handlers.append(wf_patcher.stop)
|
||||
|
||||
context.result = runner.invoke(
|
||||
session_app,
|
||||
["tell", "--session", _ULID, "--stream", "--format", "json", prompt],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the tell command returns the LLM response")
|
||||
def step_tell_returns_llm_response(context: Context) -> None:
|
||||
assert context.result.exit_code == 0, (
|
||||
f"Expected exit code 0, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
# The stub LLM response should appear in the output
|
||||
assert _STUB_RESPONSE in context.result.output, (
|
||||
f"LLM response not found in output:\n{context.result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the assistant message is persisted to the session")
|
||||
def step_assistant_message_persisted(context: Context) -> None:
|
||||
# append_message should be called exactly twice:
|
||||
# once for the user message and once for the assistant response.
|
||||
call_count = context.svc.append_message.call_count
|
||||
assert call_count == 2, (
|
||||
f"Expected append_message to be called exactly 2 times (user + assistant), "
|
||||
f"got {call_count}"
|
||||
)
|
||||
# The second call should be for the ASSISTANT role
|
||||
calls = context.svc.append_message.call_args_list
|
||||
assistant_calls = [
|
||||
c
|
||||
for c in calls
|
||||
if c.kwargs.get("role") == MessageRole.ASSISTANT
|
||||
or (len(c.args) >= 2 and c.args[1] == MessageRole.ASSISTANT)
|
||||
]
|
||||
assert len(assistant_calls) == 1, (
|
||||
f"Expected exactly 1 ASSISTANT persist, got {len(assistant_calls)}. "
|
||||
f"Calls: {calls}"
|
||||
)
|
||||
# The second call should be for the ASSISTANT role
|
||||
calls = context.svc.append_message.call_args_list
|
||||
assistant_calls = [
|
||||
c
|
||||
for c in calls
|
||||
if c.kwargs.get("role") == MessageRole.ASSISTANT
|
||||
or (len(c.args) >= 2 and c.args[1] == MessageRole.ASSISTANT)
|
||||
]
|
||||
assert len(assistant_calls) == 1, (
|
||||
f"Expected exactly 1 ASSISTANT persist, got {len(assistant_calls)}. "
|
||||
f"Calls: {calls}"
|
||||
)
|
||||
|
||||
|
||||
@then("token usage is recorded")
|
||||
def step_token_usage_recorded(context: Context) -> None:
|
||||
assert context.svc.update_token_usage.called, (
|
||||
"update_token_usage was not called — token usage was not recorded"
|
||||
)
|
||||
assert context.svc.update_token_usage.call_count == 1, (
|
||||
f"Expected update_token_usage to be called exactly once, "
|
||||
f"got {context.svc.update_token_usage.call_count}"
|
||||
)
|
||||
# Verify exact token counts match the metadata from the stub LLM (m5).
|
||||
call_kwargs = context.svc.update_token_usage.call_args.kwargs
|
||||
assert call_kwargs.get("input_tokens") == 10, (
|
||||
f"Expected input_tokens=10, got {call_kwargs.get('input_tokens')}"
|
||||
)
|
||||
assert call_kwargs.get("output_tokens") == 5, (
|
||||
f"Expected output_tokens=5, got {call_kwargs.get('output_tokens')}"
|
||||
)
|
||||
assert context.svc.update_token_usage.call_count == 1, (
|
||||
f"Expected update_token_usage to be called exactly once, "
|
||||
f"got {context.svc.update_token_usage.call_count}"
|
||||
)
|
||||
# Verify exact token counts match the metadata from the stub LLM (m5).
|
||||
call_kwargs = context.svc.update_token_usage.call_args.kwargs
|
||||
assert call_kwargs.get("input_tokens") == 10, (
|
||||
f"Expected input_tokens=10, got {call_kwargs.get('input_tokens')}"
|
||||
)
|
||||
assert call_kwargs.get("output_tokens") == 5, (
|
||||
f"Expected output_tokens=5, got {call_kwargs.get('output_tokens')}"
|
||||
)
|
||||
|
||||
|
||||
@then("the streamed output contains the LLM response tokens")
|
||||
def step_streamed_output_contains_tokens(context: Context) -> None:
|
||||
assert context.result.exit_code == 0, (
|
||||
f"Expected exit code 0, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
# Strip ANSI escape sequences for text matching
|
||||
output_clean = re.sub(r"\x1b\[[0-9;]*m", "", context.result.output)
|
||||
# When streamed through the A2A facade (Option B: fallback to
|
||||
# non-streaming), verify the full response is present in the output
|
||||
# rather than checking token-by-token word positions (C5).
|
||||
assert _STUB_RESPONSE in output_clean, (
|
||||
f"Expected full response '{_STUB_RESPONSE}' in streamed output, "
|
||||
f"but it was not found.\nOutput:\n{context.result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the assistant message is persisted after streaming")
|
||||
def step_assistant_persisted_after_stream(context: Context) -> None:
|
||||
call_count = context.svc.append_message.call_count
|
||||
assert call_count == 2, (
|
||||
f"Expected append_message to be called exactly 2 times (user + assistant), "
|
||||
f"got {call_count}"
|
||||
)
|
||||
# Verify that the last persisted message is an ASSISTANT role.
|
||||
calls = context.svc.append_message.call_args_list
|
||||
assert calls[-1].kwargs.get("role") == MessageRole.ASSISTANT, (
|
||||
f"Last persisted message role is not ASSISTANT. Calls: {calls}"
|
||||
)
|
||||
# Verify that the last persisted message is an ASSISTANT role.
|
||||
calls = context.svc.append_message.call_args_list
|
||||
assert calls[-1].kwargs.get("role") == MessageRole.ASSISTANT, (
|
||||
f"Last persisted message role is not ASSISTANT. Calls: {calls}"
|
||||
)
|
||||
|
||||
|
||||
@then("the tell command exits with code 1")
|
||||
def step_tell_exits_code_1(context: Context) -> None:
|
||||
assert context.result.exit_code == 1, (
|
||||
f"Expected exit code 1, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the error output mentions actor configuration")
|
||||
def step_error_mentions_actor(context: Context) -> None:
|
||||
output = context.result.output.lower()
|
||||
assert "actor" in output, (
|
||||
f"Expected 'actor' in error output.\nGot:\n{context.result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the tell command uses the override actor "{actor}"')
|
||||
def step_tell_uses_override_actor(context: Context, actor: str) -> None:
|
||||
invoked = getattr(context, "invoked_actor_name", None)
|
||||
assert invoked is not None, "Actor name was never captured from _resolve_llm"
|
||||
assert invoked == actor, f"Expected actor '{actor}' to be used, but got '{invoked}'"
|
||||
|
||||
|
||||
@then("the tell output is valid JSON with a data envelope")
|
||||
def step_tell_output_is_valid_json(context: Context) -> None:
|
||||
assert context.result.exit_code == 0, (
|
||||
f"Expected exit code 0, got {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
try:
|
||||
raw = json.loads(context.result.output.strip())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(
|
||||
f"Output is not valid JSON: {exc}\nOutput:\n{context.result.output}"
|
||||
) from exc
|
||||
# format_output wraps data in a spec-required envelope
|
||||
context.json_output = raw
|
||||
context.json_data = raw.get("data", {})
|
||||
assert isinstance(context.json_data, dict), (
|
||||
f"Expected data envelope to be a dict, got {type(context.json_data)}"
|
||||
)
|
||||
|
||||
|
||||
@then("the data section contains the session_id")
|
||||
def step_json_data_contains_session_id(context: Context) -> None:
|
||||
assert context.json_data.get("session_id") == _ULID, (
|
||||
f"Expected session_id={_ULID}, got {context.json_data.get('session_id')}"
|
||||
)
|
||||
|
||||
|
||||
@then("the data section contains a usage object with expected keys")
|
||||
def step_json_data_contains_usage_object(context: Context) -> None:
|
||||
usage = context.json_data.get("usage")
|
||||
assert isinstance(usage, dict), f"Expected usage to be a dict, got {type(usage)}"
|
||||
expected_keys = {
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cost_usd",
|
||||
"duration_ms",
|
||||
"tool_calls",
|
||||
}
|
||||
missing = expected_keys - set(usage.keys())
|
||||
assert not missing, f"Usage object missing expected keys: {missing}\nUsage: {usage}"
|
||||
|
||||
|
||||
@then("the output contains a Usage panel")
|
||||
def step_output_contains_usage_panel(context: Context) -> None:
|
||||
output = context.result.output
|
||||
# Rich Usage panel is rendered with title="Usage"
|
||||
assert "Usage" in output, f"Expected Usage panel in output.\nOutput:\n{output}"
|
||||
assert "Input tokens" in output, (
|
||||
f"Expected input tokens in Usage panel.\nOutput:\n{output}"
|
||||
)
|
||||
assert "Output tokens" in output, (
|
||||
f"Expected output tokens in Usage panel.\nOutput:\n{output}"
|
||||
)
|
||||
@@ -0,0 +1,473 @@
|
||||
"""Step definitions for session_workflow.py coverage boost tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.application.services.session_caller import (
|
||||
LangChainSessionCaller,
|
||||
_MinimalStubLLM,
|
||||
estimate_cost,
|
||||
extract_content,
|
||||
extract_token_usage,
|
||||
history_to_langchain_messages,
|
||||
)
|
||||
from cleveragents.application.services.session_workflow import (
|
||||
SessionWorkflow,
|
||||
)
|
||||
from cleveragents.domain.models.core.session import (
|
||||
MessageRole,
|
||||
SessionMessage,
|
||||
)
|
||||
from cleveragents.tool.actor_runtime import LLMResponse
|
||||
|
||||
|
||||
@given("the session workflow coverage environment is set up")
|
||||
def step_sw_coverage_setup(context: Context) -> None:
|
||||
"""No special setup needed."""
|
||||
pass
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _extract_content
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given('coverage boost a mock response with content "hello world"')
|
||||
def step_cb_mock_content(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.content = "hello world"
|
||||
resp.text = "ignored"
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given('coverage boost a mock response with text "from text attr" and no content')
|
||||
def step_cb_text_fallback(context: Context) -> None:
|
||||
class _TextOnly:
|
||||
def __init__(self) -> None:
|
||||
self.text = "from text attr"
|
||||
|
||||
context.cb_mock_response = _TextOnly()
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a mock response with list content containing text dicts and plain strings"
|
||||
)
|
||||
def step_cb_list_content(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.content = [
|
||||
{"text": "hello"},
|
||||
"world",
|
||||
{"content": "test"},
|
||||
]
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given("coverage boost a mock response with no content or text attribute")
|
||||
def step_cb_no_attrs(context: Context) -> None:
|
||||
class _Unknown:
|
||||
pass
|
||||
|
||||
context.cb_mock_response = _Unknown()
|
||||
|
||||
|
||||
@when("coverage boost _extract_content is called")
|
||||
def step_cb_call_extract_content(context: Context) -> None:
|
||||
context.cb_extract_result = extract_content(context.cb_mock_response)
|
||||
|
||||
|
||||
@then('coverage boost the extracted result should be "{expected}"')
|
||||
def step_cb_extract_result_is(context: Context, expected: str) -> None:
|
||||
assert context.cb_extract_result == expected, (
|
||||
f"Expected {expected!r}, got {context.cb_extract_result!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("coverage boost the result should contain the concatenated texts")
|
||||
def step_cb_result_concat(context: Context) -> None:
|
||||
assert "hello" in context.cb_extract_result
|
||||
assert "world" in context.cb_extract_result
|
||||
assert "test" in context.cb_extract_result
|
||||
|
||||
|
||||
@then("coverage boost the result should be a string")
|
||||
def step_cb_result_str(context: Context) -> None:
|
||||
assert isinstance(context.cb_extract_result, str)
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _extract_token_usage
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a mock response with response_metadata usage input_tokens=100 output_tokens=50"
|
||||
)
|
||||
def step_cb_usage_metadata(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.response_metadata = {"usage": {"input_tokens": 100, "output_tokens": 50}}
|
||||
resp.usage_metadata = {}
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a mock response with response_metadata token_usage input_tokens=200 output_tokens=100"
|
||||
)
|
||||
def step_cb_token_usage_metadata(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.response_metadata = {
|
||||
"token_usage": {"input_tokens": 200, "output_tokens": 100}
|
||||
}
|
||||
resp.usage_metadata = {}
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a mock response with response_metadata usage prompt_tokens=300 completion_tokens=150"
|
||||
)
|
||||
def step_cb_prompt_completion(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.response_metadata = {"usage": {"prompt_tokens": 300, "completion_tokens": 150}}
|
||||
resp.usage_metadata = {}
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a mock response with usage_metadata input_tokens=400 output_tokens=200"
|
||||
)
|
||||
def step_cb_usage_meta_attr(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.response_metadata = {}
|
||||
resp.usage_metadata = {"input_tokens": 400, "output_tokens": 200}
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@given("coverage boost a mock response with no usage metadata")
|
||||
def step_cb_no_metadata(context: Context) -> None:
|
||||
resp = MagicMock()
|
||||
resp.response_metadata = {}
|
||||
resp.usage_metadata = {}
|
||||
context.cb_mock_response = resp
|
||||
|
||||
|
||||
@when("coverage boost _extract_token_usage is called")
|
||||
def step_cb_call_extract_token_usage(context: Context) -> None:
|
||||
context.cb_token_input, context.cb_token_output = extract_token_usage(
|
||||
context.cb_mock_response
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
"coverage boost input tokens should be {input_tokens:d} and output tokens should be {output_tokens:d}"
|
||||
)
|
||||
def step_cb_assert_token_counts(
|
||||
context: Context, input_tokens: int, output_tokens: int
|
||||
) -> None:
|
||||
assert context.cb_token_input == input_tokens, (
|
||||
f"Expected {input_tokens} input tokens, got {context.cb_token_input}"
|
||||
)
|
||||
assert context.cb_token_output == output_tokens, (
|
||||
f"Expected {output_tokens} output tokens, got {context.cb_token_output}"
|
||||
)
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _estimate_cost
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost input tokens {input_tokens:d} and output tokens {output_tokens:d}"
|
||||
)
|
||||
def step_cb_given_tokens(
|
||||
context: Context, input_tokens: int, output_tokens: int
|
||||
) -> None:
|
||||
context.cb_input_tokens = input_tokens
|
||||
context.cb_output_tokens = output_tokens
|
||||
|
||||
|
||||
@when("coverage boost _estimate_cost is called")
|
||||
def step_cb_call_estimate_cost(context: Context) -> None:
|
||||
context.cb_estimated_cost = estimate_cost(
|
||||
context.cb_input_tokens, context.cb_output_tokens
|
||||
)
|
||||
|
||||
|
||||
@then("coverage boost the estimated cost should be positive")
|
||||
def step_cb_cost_positive(context: Context) -> None:
|
||||
assert context.cb_estimated_cost > 0.0, (
|
||||
f"Expected positive cost, got {context.cb_estimated_cost}"
|
||||
)
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _history_to_langchain_messages
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given("coverage boost session messages with roles SYSTEM, USER, ASSISTANT, and TOOL")
|
||||
def step_cb_messages_all_roles(context: Context) -> None:
|
||||
context.cb_history_messages = [
|
||||
SessionMessage(
|
||||
message_id="01KJ1N8YDN05N29P5RZV4SPDVZ",
|
||||
role=MessageRole.SYSTEM,
|
||||
content="You are helpful.",
|
||||
sequence=1,
|
||||
timestamp="2026-01-01T00:00:00Z",
|
||||
),
|
||||
SessionMessage(
|
||||
message_id="01KJ1N8YDN05N29P5RZV4SPDW0",
|
||||
role=MessageRole.USER,
|
||||
content="Hello",
|
||||
sequence=2,
|
||||
timestamp="2026-01-01T00:00:00Z",
|
||||
),
|
||||
SessionMessage(
|
||||
message_id="01KJ1N8YDN05N29P5RZV4SPDW1",
|
||||
role=MessageRole.ASSISTANT,
|
||||
content="Hi!",
|
||||
sequence=3,
|
||||
timestamp="2026-01-01T00:00:00Z",
|
||||
),
|
||||
SessionMessage(
|
||||
message_id="01KJ1N8YDN05N29P5RZV4SPDW2",
|
||||
role=MessageRole.TOOL,
|
||||
content="result",
|
||||
sequence=4,
|
||||
timestamp="2026-01-01T00:00:00Z",
|
||||
tool_call_id="tc1",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@when("coverage boost _history_to_langchain_messages is called")
|
||||
def step_cb_call_history_to_lc(context: Context) -> None:
|
||||
context.cb_lc_messages = history_to_langchain_messages(context.cb_history_messages)
|
||||
|
||||
|
||||
@then(
|
||||
"coverage boost the result should contain SystemMessage, HumanMessage, AIMessage, and ToolMessage"
|
||||
)
|
||||
def step_cb_assert_lc_types(context: Context) -> None:
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
|
||||
from langchain_core.messages.tool import ToolMessage
|
||||
|
||||
types = [type(m) for m in context.cb_lc_messages]
|
||||
assert SystemMessage in types, f"Missing SystemMessage in {types}"
|
||||
assert HumanMessage in types, f"Missing HumanMessage in {types}"
|
||||
assert AIMessage in types, f"Missing AIMessage in {types}"
|
||||
assert ToolMessage in types, f"Missing ToolMessage in {types}"
|
||||
|
||||
|
||||
@given("coverage boost a session message with an unknown role")
|
||||
def step_cb_message_unknown_role(context: Context) -> None:
|
||||
msg = MagicMock(spec=SessionMessage)
|
||||
msg.role = "NOT_A_REAL_ROLE"
|
||||
msg.content = "test content"
|
||||
msg.tool_call_id = None
|
||||
context.cb_history_messages = [msg]
|
||||
|
||||
|
||||
@then("coverage boost the result should contain a HumanMessage")
|
||||
def step_cb_result_contains_human(context: Context) -> None:
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
assert len(context.cb_lc_messages) > 0, "Expected at least one message"
|
||||
assert isinstance(context.cb_lc_messages[0], HumanMessage), (
|
||||
f"Expected HumanMessage, got {type(context.cb_lc_messages[0])}"
|
||||
)
|
||||
|
||||
|
||||
@given("coverage boost an empty list of session messages")
|
||||
def step_cb_empty_history(context: Context) -> None:
|
||||
context.cb_history_messages = []
|
||||
|
||||
|
||||
@then("coverage boost the result should be an empty list")
|
||||
def step_cb_empty_list(context: Context) -> None:
|
||||
assert context.cb_lc_messages == [], (
|
||||
f"Expected empty list, got {context.cb_lc_messages}"
|
||||
)
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# LangChainSessionCaller.invoke() tool_results
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given("coverage boost a LangChainSessionCaller with a stub LLM and empty history")
|
||||
def step_cb_caller_with_stub(context: Context) -> None:
|
||||
stub_llm = MagicMock()
|
||||
stub_llm.invoke.return_value = MagicMock(
|
||||
content="response text",
|
||||
tool_calls=[],
|
||||
response_metadata={},
|
||||
usage_metadata={},
|
||||
)
|
||||
context.cb_caller = LangChainSessionCaller(llm=stub_llm, history=[])
|
||||
context.cb_stub_llm = stub_llm
|
||||
|
||||
|
||||
@when(
|
||||
"coverage boost invoke is called with tool_results containing one success and one failure"
|
||||
)
|
||||
def step_cb_invoke_tool_results(context: Context) -> None:
|
||||
context.cb_caller.invoke(prompt="test prompt", tool_schemas=[])
|
||||
context.cb_tool_result_response = context.cb_caller.invoke(
|
||||
prompt="test prompt",
|
||||
tool_schemas=[],
|
||||
tool_results=[
|
||||
{
|
||||
"success": True,
|
||||
"output": {"result": "ok"},
|
||||
"call_id": "c1",
|
||||
"tool_name": "tool1",
|
||||
},
|
||||
{
|
||||
"success": False,
|
||||
"error": "failed",
|
||||
"call_id": "c2",
|
||||
"tool_name": "tool2",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@then("coverage boost the accumulated messages should include tool result messages")
|
||||
def step_cb_accumulated_has_tool_results(context: Context) -> None:
|
||||
from langchain_core.messages.tool import ToolMessage
|
||||
|
||||
tool_msgs = [
|
||||
m for m in context.cb_caller._accumulated if isinstance(m, ToolMessage)
|
||||
]
|
||||
assert len(tool_msgs) >= 2, f"Expected >= 2 ToolMessages, got {len(tool_msgs)}"
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# LangChainSessionCaller.invoke() with tool_calls in response
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given(
|
||||
"coverage boost a LangChainSessionCaller with a stub LLM that returns tool calls"
|
||||
)
|
||||
def step_cb_caller_tool_calls(context: Context) -> None:
|
||||
stub_llm = MagicMock()
|
||||
resp = MagicMock(
|
||||
content="response with tool calls",
|
||||
tool_calls=[
|
||||
{"name": "test_tool", "args": {"arg1": "val1"}, "id": "call_123"},
|
||||
],
|
||||
response_metadata={},
|
||||
usage_metadata={},
|
||||
)
|
||||
stub_llm.invoke.return_value = resp
|
||||
context.cb_caller = LangChainSessionCaller(llm=stub_llm, history=[])
|
||||
|
||||
|
||||
@when("coverage boost invoke is called for the first time")
|
||||
def step_cb_invoke_first_time(context: Context) -> None:
|
||||
context.cb_first_invoke_result = context.cb_caller.invoke(
|
||||
prompt="test prompt", tool_schemas=[]
|
||||
)
|
||||
|
||||
|
||||
@then("coverage boost the LLMResponse should contain the extracted tool calls")
|
||||
def step_cb_llm_response_has_tool_calls(context: Context) -> None:
|
||||
assert isinstance(context.cb_first_invoke_result, LLMResponse)
|
||||
assert len(context.cb_first_invoke_result.tool_calls) >= 1, (
|
||||
f"Expected >= 1 tool calls, got {len(context.cb_first_invoke_result.tool_calls)}"
|
||||
)
|
||||
tc = context.cb_first_invoke_result.tool_calls[0]
|
||||
assert tc.name == "test_tool"
|
||||
assert tc.arguments == {"arg1": "val1"}
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _build_lc_messages_from_history (SessionWorkflow method)
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given("coverage boost a SessionWorkflow with a stub service and no registry")
|
||||
def step_cb_workflow_with_stub(context: Context) -> None:
|
||||
svc = MagicMock()
|
||||
svc.get.return_value = MagicMock(actor_name="openai/gpt-4")
|
||||
svc.get_messages.return_value = []
|
||||
svc.append_message.return_value = MagicMock(spec=SessionMessage)
|
||||
svc.update_token_usage.return_value = None
|
||||
context.cb_workflow = SessionWorkflow(session_service=svc, provider_registry=None)
|
||||
|
||||
|
||||
@given("coverage boost session history without a system message")
|
||||
def step_cb_history_no_system(context: Context) -> None:
|
||||
context.cb_no_system_history = [
|
||||
SessionMessage(
|
||||
message_id="01KJ1N8YDN05N29P5RZV4SPDW0",
|
||||
role=MessageRole.USER,
|
||||
content="Hello",
|
||||
sequence=1,
|
||||
timestamp="2026-01-01T00:00:00Z",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@when("coverage boost _build_lc_messages_from_history is called with a prompt")
|
||||
def step_cb_call_build_lc(context: Context) -> None:
|
||||
context.cb_built_messages = context.cb_workflow._build_lc_messages_from_history(
|
||||
context.cb_no_system_history, "test prompt"
|
||||
)
|
||||
|
||||
|
||||
@then(
|
||||
"coverage boost the first message should be a SystemMessage with the session system prompt"
|
||||
)
|
||||
def step_cb_first_is_system(context: Context) -> None:
|
||||
from langchain_core.messages import SystemMessage
|
||||
|
||||
assert len(context.cb_built_messages) > 0, "Expected at least one message"
|
||||
first = context.cb_built_messages[0]
|
||||
assert isinstance(first, SystemMessage), (
|
||||
f"Expected SystemMessage, got {type(first)}"
|
||||
)
|
||||
assert "CleverAgents orchestrator" in first.content, (
|
||||
f"Expected system prompt content, got {first.content}"
|
||||
)
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# _MinimalStubLLM
|
||||
# ====================================================================
|
||||
|
||||
|
||||
@given("coverage boost a _MinimalStubLLM instance")
|
||||
def step_cb_minimal_stub(context: Context) -> None:
|
||||
context.cb_stub = _MinimalStubLLM()
|
||||
|
||||
|
||||
@when("coverage boost invoke on the stub is called")
|
||||
def step_cb_stub_invoke(context: Context) -> None:
|
||||
context.cb_stub_response = context.cb_stub.invoke([])
|
||||
|
||||
|
||||
@then('coverage boost the stub response content should be "(no LLM configured)"')
|
||||
def step_cb_stub_content(context: Context) -> None:
|
||||
assert context.cb_stub_response.content == "(no LLM configured)"
|
||||
|
||||
|
||||
@then("coverage boost the stub response should have empty tool_calls")
|
||||
def step_cb_stub_empty_tool_calls(context: Context) -> None:
|
||||
assert context.cb_stub_response.tool_calls == []
|
||||
|
||||
|
||||
@when("coverage boost stream on the stub is called")
|
||||
def step_cb_stub_stream(context: Context) -> None:
|
||||
context.cb_stub_stream_chunks = list(context.cb_stub.stream([]))
|
||||
|
||||
|
||||
@then('coverage boost it should yield a chunk with content "(no LLM configured)"')
|
||||
def step_cb_stub_chunk_content(context: Context) -> None:
|
||||
assert len(context.cb_stub_stream_chunks) >= 1, "Expected at least one chunk"
|
||||
assert context.cb_stub_stream_chunks[0].content == "(no LLM configured)"
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Step definitions for Strategize decision recording feature.
|
||||
|
||||
Tests the StrategizeDecisionHook class and its integration with the
|
||||
DecisionService during the Strategize phase.
|
||||
|
||||
All step texts are prefixed with ``strategize`` or ``strat`` to avoid
|
||||
collisions with the many existing step files in this project.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.application.services.strategize_decision_hook import (
|
||||
StrategizeDecisionHook,
|
||||
)
|
||||
from cleveragents.core.exceptions import ValidationError
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a strategize decision service")
|
||||
def step_given_strategize_decision_service(context):
|
||||
"""Create an in-memory decision service for Strategize tests."""
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
|
||||
@given('a strategize decision hook for plan "{plan_id}"')
|
||||
def step_given_strategize_hook(context, plan_id):
|
||||
"""Create a strategize decision hook for the given plan."""
|
||||
context.plan_id = plan_id
|
||||
context.hook = StrategizeDecisionHook(
|
||||
decision_service=context.decision_service,
|
||||
plan_id=plan_id,
|
||||
)
|
||||
context.last_decision = None
|
||||
context.first_decision_id = None
|
||||
context.context_data = None
|
||||
context.actor_state = None
|
||||
context.relevant_resources = None
|
||||
context.alternatives = None
|
||||
context.confidence = None
|
||||
context.rationale = None
|
||||
context.parent_decision_id = None
|
||||
context.error = None
|
||||
context.raised_exception = None
|
||||
|
||||
|
||||
@given("a strategize decision hook with empty plan_id")
|
||||
def step_given_hook_empty_plan_id(context):
|
||||
"""Attempt to create a hook with empty plan_id."""
|
||||
context.error = None
|
||||
try:
|
||||
context.hook = StrategizeDecisionHook(
|
||||
decision_service=context.decision_service,
|
||||
plan_id="",
|
||||
)
|
||||
except ValidationError as e:
|
||||
context.error = e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategy choice recording steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I record a strategy choice with question "{question}" and option "{option}"')
|
||||
def step_when_record_strategy_choice(context, question, option):
|
||||
"""Record a strategy choice decision."""
|
||||
context.last_decision = context.hook.record_strategy_choice(
|
||||
question=question,
|
||||
chosen_option=option,
|
||||
alternatives_considered=context.alternatives,
|
||||
confidence_score=context.confidence,
|
||||
rationale=context.rationale or "",
|
||||
context_data=context.context_data,
|
||||
actor_state=context.actor_state,
|
||||
relevant_resources=context.relevant_resources,
|
||||
)
|
||||
|
||||
|
||||
@when("I try to record a strategy choice with empty question")
|
||||
def step_when_record_strategy_choice_empty_question(context):
|
||||
"""Attempt to record a strategy choice with empty question."""
|
||||
context.error = None
|
||||
try:
|
||||
context.hook.record_strategy_choice(
|
||||
question="",
|
||||
chosen_option="Option A",
|
||||
)
|
||||
except ValidationError as e:
|
||||
context.error = e
|
||||
|
||||
|
||||
@when("I try to record a strategy choice with empty chosen_option")
|
||||
def step_when_record_strategy_choice_empty_option(context):
|
||||
"""Attempt to record a strategy choice with empty option."""
|
||||
context.error = None
|
||||
try:
|
||||
context.hook.record_strategy_choice(
|
||||
question="Which approach?",
|
||||
chosen_option="",
|
||||
)
|
||||
except ValidationError as e:
|
||||
context.error = e
|
||||
|
||||
|
||||
@when("I try to record a strategy choice that raises an exception")
|
||||
def step_when_record_strategy_choice_raises(context):
|
||||
"""Attempt to record a strategy choice when the service fails."""
|
||||
context.raised_exception = None
|
||||
try:
|
||||
context.hook.record_strategy_choice(
|
||||
question="Which approach?",
|
||||
chosen_option="Approach A",
|
||||
)
|
||||
except Exception as exc:
|
||||
context.raised_exception = exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resource selection recording steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I record a resource selection with question "{question}" and option "{option}"')
|
||||
def step_when_record_resource_selection(context, question, option):
|
||||
"""Record a resource selection decision."""
|
||||
context.last_decision = context.hook.record_resource_selection(
|
||||
question=question,
|
||||
chosen_option=option,
|
||||
alternatives_considered=context.alternatives,
|
||||
confidence_score=context.confidence,
|
||||
rationale=context.rationale or "",
|
||||
context_data=context.context_data,
|
||||
actor_state=context.actor_state,
|
||||
relevant_resources=context.relevant_resources,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subplan spawn recording steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I record a subplan spawn with question "{question}" and option "{option}"')
|
||||
def step_when_record_subplan_spawn(context, question, option):
|
||||
"""Record a subplan spawn decision."""
|
||||
context.last_decision = context.hook.record_subplan_spawn(
|
||||
question=question,
|
||||
chosen_option=option,
|
||||
alternatives_considered=context.alternatives,
|
||||
confidence_score=context.confidence,
|
||||
rationale=context.rationale or "",
|
||||
context_data=context.context_data,
|
||||
actor_state=context.actor_state,
|
||||
relevant_resources=context.relevant_resources,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invariant enforcement recording steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I record an invariant enforced with question "{question}" and option "{option}"')
|
||||
def step_when_record_invariant_enforced(context, question, option):
|
||||
"""Record an invariant enforced decision."""
|
||||
context.last_decision = context.hook.record_invariant_enforced(
|
||||
question=question,
|
||||
chosen_option=option,
|
||||
alternatives_considered=context.alternatives,
|
||||
confidence_score=context.confidence,
|
||||
rationale=context.rationale or "",
|
||||
context_data=context.context_data,
|
||||
actor_state=context.actor_state,
|
||||
relevant_resources=context.relevant_resources,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Context data steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('two strat alternatives "{alt1}" and "{alt2}"')
|
||||
def step_when_alternatives(context, alt1, alt2):
|
||||
"""Set two alternatives for the next decision."""
|
||||
context.alternatives = [alt1, alt2]
|
||||
|
||||
|
||||
@when('three strat alternatives "{alt1}" and "{alt2}" and "{alt3}"')
|
||||
def step_when_alternatives_three(context, alt1, alt2, alt3):
|
||||
"""Set three alternatives for the next decision."""
|
||||
context.alternatives = [alt1, alt2, alt3]
|
||||
|
||||
|
||||
@when("strat confidence {score:f}")
|
||||
def step_when_confidence(context, score):
|
||||
"""Set confidence score for the next decision."""
|
||||
context.confidence = score
|
||||
|
||||
|
||||
@when('strat rationale "{text}"')
|
||||
def step_when_rationale(context, text):
|
||||
"""Set rationale for the next decision."""
|
||||
context.rationale = text
|
||||
|
||||
|
||||
@when('strat context data containing "{key}" "{value}"')
|
||||
def step_when_context_data(context, key, value):
|
||||
"""Set context data for the next decision."""
|
||||
context.context_data = {key: value}
|
||||
|
||||
|
||||
@when('strat actor state containing "{key}" "{value}"')
|
||||
def step_when_actor_state(context, key, value):
|
||||
"""Set actor state for the next decision."""
|
||||
context.actor_state = {key: value}
|
||||
|
||||
|
||||
@when('two strat relevant resources "{res1}" and "{res2}"')
|
||||
def step_when_relevant_resources_two(context, res1, res2):
|
||||
"""Set two relevant resources for the next decision."""
|
||||
context.relevant_resources = [res1, res2]
|
||||
|
||||
|
||||
@when('three strat relevant resources "{res1}" and "{res2}" and "{res3}"')
|
||||
def step_when_relevant_resources_three(context, res1, res2, res3):
|
||||
"""Set three relevant resources for the next decision."""
|
||||
context.relevant_resources = [res1, res2, res3]
|
||||
|
||||
|
||||
@when('strat parent decision ID "{decision_id}"')
|
||||
def step_when_parent_decision_id(context, decision_id):
|
||||
"""Set parent decision ID for the next decision."""
|
||||
context.parent_decision_id = decision_id
|
||||
# Recreate hook with parent ID
|
||||
context.hook = StrategizeDecisionHook(
|
||||
decision_service=context.decision_service,
|
||||
plan_id=context.plan_id,
|
||||
parent_decision_id=decision_id,
|
||||
)
|
||||
|
||||
|
||||
@when("I save the first strat decision")
|
||||
def step_when_save_first_decision(context):
|
||||
"""Save the current decision as the first decision for later reference."""
|
||||
assert context.last_decision is not None, "No decision recorded yet"
|
||||
context.first_decision_id = context.last_decision.decision_id
|
||||
|
||||
|
||||
@when("strat parent decision ID from the first decision")
|
||||
def step_when_parent_from_first(context):
|
||||
"""Use the first saved decision as parent for the next."""
|
||||
assert context.first_decision_id is not None, "No first decision saved"
|
||||
context.parent_decision_id = context.first_decision_id
|
||||
context.hook = StrategizeDecisionHook(
|
||||
decision_service=context.decision_service,
|
||||
plan_id=context.plan_id,
|
||||
parent_decision_id=context.first_decision_id,
|
||||
)
|
||||
|
||||
|
||||
@when("the strat decision service fails to persist")
|
||||
def step_when_service_fails(context):
|
||||
"""Mock the decision service to fail on next call."""
|
||||
original_record = context.decision_service.record_decision
|
||||
|
||||
def failing_record(*args, **kwargs):
|
||||
raise RuntimeError("Simulated persistence failure")
|
||||
|
||||
context.decision_service.record_decision = failing_record
|
||||
context.original_record = original_record
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assertion steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the strat decision should be recorded successfully")
|
||||
def step_then_decision_recorded(context):
|
||||
"""Verify the decision was recorded."""
|
||||
assert context.last_decision is not None
|
||||
assert context.last_decision.decision_id is not None
|
||||
assert context.last_decision.plan_id == context.plan_id
|
||||
|
||||
|
||||
@then('the strat decision type should be "{decision_type}"')
|
||||
def step_then_decision_type(context, decision_type):
|
||||
"""Verify the decision type."""
|
||||
assert context.last_decision.decision_type.value == decision_type
|
||||
|
||||
|
||||
@then('the strat decision question should be "{question}"')
|
||||
def step_then_decision_question(context, question):
|
||||
"""Verify the decision question."""
|
||||
assert context.last_decision.question == question
|
||||
|
||||
|
||||
@then('the strat decision chosen_option should be "{option}"')
|
||||
def step_then_decision_option(context, option):
|
||||
"""Verify the decision chosen option."""
|
||||
assert context.last_decision.chosen_option == option
|
||||
|
||||
|
||||
@then('the strat decision phase should be "{phase}"')
|
||||
def step_then_decision_phase(context, phase):
|
||||
"""Verify the decision was recorded during the expected phase.
|
||||
|
||||
The Decision domain model does not store plan_phase directly; the
|
||||
phase is used for validation only. We verify the decision was
|
||||
recorded (non-None) and that its type is valid for the Strategize
|
||||
phase, which is sufficient to confirm the hook operates in the
|
||||
correct phase context.
|
||||
"""
|
||||
assert context.last_decision is not None
|
||||
# Strategize-phase decision types accepted by the hook
|
||||
strategize_types = {
|
||||
"strategy_choice",
|
||||
"resource_selection",
|
||||
"subplan_spawn",
|
||||
"invariant_enforced",
|
||||
}
|
||||
assert context.last_decision.decision_type.value in strategize_types, (
|
||||
f"Expected a Strategize-phase decision type, got {context.last_decision.decision_type.value!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the strat decision should have {count:d} alternatives considered")
|
||||
def step_then_alternatives_count(context, count):
|
||||
"""Verify the number of alternatives."""
|
||||
assert len(context.last_decision.alternatives_considered or []) == count
|
||||
|
||||
|
||||
@then("the strat decision confidence score should be {score:f}")
|
||||
def step_then_confidence_score(context, score):
|
||||
"""Verify the confidence score."""
|
||||
assert context.last_decision.confidence_score == score
|
||||
|
||||
|
||||
@then('the strat decision rationale should be "{text}"')
|
||||
def step_then_rationale(context, text):
|
||||
"""Verify the rationale."""
|
||||
assert context.last_decision.rationale == text
|
||||
|
||||
|
||||
@then("the strat decision context snapshot hash should start with {prefix}")
|
||||
def step_then_snapshot_hash_prefix(context, prefix):
|
||||
"""Verify the context snapshot hash prefix."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
assert snapshot.hot_context_hash.startswith(prefix.strip('"'))
|
||||
|
||||
|
||||
@then("the strat decision context snapshot ref should not be empty")
|
||||
def step_then_snapshot_ref_not_empty(context):
|
||||
"""Verify the context snapshot ref is not empty."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
assert snapshot.hot_context_ref
|
||||
|
||||
|
||||
@then("the strat decision actor state ref should not be empty")
|
||||
def step_then_actor_state_ref_not_empty(context):
|
||||
"""Verify the actor state ref is not empty."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
assert snapshot.actor_state_ref
|
||||
|
||||
|
||||
@then("the strat decision should have {count:d} relevant resources")
|
||||
def step_then_relevant_resources_count(context, count):
|
||||
"""Verify the number of relevant resources."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
assert len(snapshot.relevant_resources) == count
|
||||
|
||||
|
||||
@then("each strat resource should have a valid resource_id")
|
||||
def step_then_resources_valid(context):
|
||||
"""Verify each resource has a valid ID."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
for resource in snapshot.relevant_resources:
|
||||
assert resource.resource_id
|
||||
assert len(resource.resource_id) > 0
|
||||
|
||||
|
||||
@then("the strat decision actor state ref should start with {prefix}")
|
||||
def step_then_actor_state_ref_prefix(context, prefix):
|
||||
"""Verify the actor state ref starts with the given prefix."""
|
||||
snapshot = context.last_decision.context_snapshot
|
||||
assert snapshot is not None
|
||||
assert snapshot.actor_state_ref.startswith(prefix.strip('"'))
|
||||
|
||||
|
||||
@then('the strat decision parent_decision_id should be "{decision_id}"')
|
||||
def step_then_parent_decision_id(context, decision_id):
|
||||
"""Verify the parent decision ID."""
|
||||
assert context.last_decision.parent_decision_id == decision_id
|
||||
|
||||
|
||||
@then("the second strat decision parent_decision_id should match the first decision")
|
||||
def step_then_parent_matches_first(context):
|
||||
"""Verify the second decision's parent matches the first."""
|
||||
assert context.last_decision.parent_decision_id == context.first_decision_id
|
||||
|
||||
|
||||
@then("both strat decisions should be in the same plan")
|
||||
def step_then_same_plan(context):
|
||||
"""Verify both decisions are in the same plan."""
|
||||
assert context.last_decision.plan_id == context.plan_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error handling steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("a strat validation error should be raised")
|
||||
def step_then_validation_error(context):
|
||||
"""Verify a validation error was raised."""
|
||||
assert context.error is not None
|
||||
assert isinstance(context.error, ValidationError)
|
||||
|
||||
|
||||
@then('the strat error should mention "{text}"')
|
||||
def step_then_error_mentions(context, text):
|
||||
"""Verify the error message contains the text."""
|
||||
assert text in str(context.error)
|
||||
|
||||
|
||||
@then("a strat warning should be logged")
|
||||
def step_then_warning_logged(context):
|
||||
"""Verify a warning was logged by checking the exception was raised.
|
||||
|
||||
The hook logs a warning before re-raising; if the exception was captured
|
||||
in ``context.raised_exception`` the warning path was exercised.
|
||||
"""
|
||||
assert context.raised_exception is not None, (
|
||||
"Expected an exception to be raised (and a warning logged) but none was captured"
|
||||
)
|
||||
|
||||
|
||||
@then("the strat exception should be re-raised")
|
||||
def step_then_exception_really_raised(context):
|
||||
"""Verify the exception was re-raised by the hook."""
|
||||
assert context.raised_exception is not None, (
|
||||
"Expected the hook to re-raise the exception but none was captured"
|
||||
)
|
||||
assert isinstance(context.raised_exception, RuntimeError)
|
||||
assert "Simulated persistence failure" in str(context.raised_exception)
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Step definitions for structural component output validation.
|
||||
|
||||
Tests for features/structural_validation.feature - validates all four
|
||||
validators covering plan tree nodes, decision CLI dicts, structured
|
||||
output envelopes, and the unified dispatcher.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when # type: ignore[import-untyped]
|
||||
from behave.runner import Context # type: ignore[import-untyped]
|
||||
|
||||
from cleveragents.core.validation import (
|
||||
ValidationError,
|
||||
validate_decision_dict,
|
||||
validate_plan_tree,
|
||||
validate_structured_component_output,
|
||||
validate_structured_output,
|
||||
)
|
||||
|
||||
# Helper: valid ULID strings for test data
|
||||
VALID_ULID_A = "01ARZ3NDEKTSV4XXFFJFRC889A"
|
||||
VALID_ULID_B = "01ARZ3NDEKTSV4XXFFJFRC889B"
|
||||
VALID_ULID_C = "01ARZ3NDEKTSV4XXFFJFRC889C"
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Plan tree scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('the plan tree contains {node_count:d} node(s)')
|
||||
def step_plan_tree_node_count(ctx: Context, node_count: int) -> None:
|
||||
"""Create a list of valid plan tree nodes."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}",
|
||||
"type": "strategy_choice",
|
||||
"sequence": i,
|
||||
"question": f"Decision question #{i}",
|
||||
"children": [],
|
||||
**(
|
||||
{"parent_decision_id": VALID_ULID_A}
|
||||
if i > 0 else {}
|
||||
),
|
||||
}
|
||||
for i in range(node_count)
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node missing "decision_id"')
|
||||
def step_node_missing_decision_id(ctx: Context) -> None:
|
||||
"""Create a node missing the required decision_id field."""
|
||||
ctx.tree_nodes = [
|
||||
{"type": "strategy_choice", "sequence": 0, "question": "Missing ID", "children": []}
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node with invalid ULID "not-a-ulid-12345"')
|
||||
def step_node_invalid_ulid(ctx: Context) -> None:
|
||||
"""Create a node with an invalid decision_id."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": "not-a-ulid-12345",
|
||||
"type": "strategy_choice",
|
||||
"sequence": 0,
|
||||
"question": "Invalid ULID test",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@given('the plan tree contains a node with empty "children" list')
|
||||
def step_node_empty_children(ctx: Context) -> None:
|
||||
"""Create a node with an empty children list - should be valid."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 0,
|
||||
"question": "Question text",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@given('two sibling nodes share the same sequence number')
|
||||
def step_duplicate_sequences(ctx: Context) -> None:
|
||||
"""Create two sibling nodes with identical sequence."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 5,
|
||||
"question": "First",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_B,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_C,
|
||||
"type": "implementation_choice",
|
||||
"sequence": 5,
|
||||
"question": "Second",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_B,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@given('a plan tree node with sequence "{seq}"')
|
||||
def step_node_sequence(ctx: Context, seq: str) -> None:
|
||||
"""Create a node with the given sequence value (as string to parse)."""
|
||||
int_seq = int(seq)
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "strategy_choice",
|
||||
"sequence": int_seq,
|
||||
"question": "Q",
|
||||
"children": [],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Decision dict scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_value(raw: str) -> Any:
|
||||
"""Parse a behave value string into Python type."""
|
||||
stripped = raw.strip().strip('"').strip("'")
|
||||
if stripped in ("null", "none", "None"):
|
||||
return None
|
||||
if stripped == "true":
|
||||
return True
|
||||
if stripped == "false":
|
||||
return False
|
||||
try:
|
||||
return int(stripped)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(stripped)
|
||||
except ValueError:
|
||||
pass
|
||||
return stripped
|
||||
|
||||
|
||||
def _ensure_decision_dict(ctx: Context) -> dict:
|
||||
"""Ensure ctx.decision_dict exists with defaults."""
|
||||
if not hasattr(ctx, "decision_dict"):
|
||||
ctx.decision_dict = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Which approach?",
|
||||
"chosen": "Option A",
|
||||
"confidence": 0.75,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
return ctx.decision_dict # type: ignore[return-value]
|
||||
|
||||
|
||||
@given('I have a decision dict with "{field}" set to {value}')
|
||||
def step_decision_dict_field(ctx: Context, field: str, value) -> Any: # type: ignore[no-untyped-def]
|
||||
"""Set a single field on the decision dict."""
|
||||
d = _ensure_decision_dict(ctx)
|
||||
d[field] = _parse_value(str(value))
|
||||
|
||||
|
||||
@given('I have a valid decision dict')
|
||||
@when('I validate the decision dict')
|
||||
def step_validate_decision(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
d = _ensure_decision_dict(ctx)
|
||||
ctx.decision_result = validate_decision_dict(d)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Structured output scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
def _ensure_struct_output(ctx: Context) -> dict:
|
||||
"""Ensure ctx.struct_output exists with defaults."""
|
||||
if not hasattr(ctx, "struct_output"):
|
||||
ctx.struct_output = {
|
||||
"command": "agents plan list",
|
||||
"session_id": VALID_ULID_A,
|
||||
"status": "ok",
|
||||
"exit_code": 0,
|
||||
"elements": [],
|
||||
}
|
||||
return ctx.struct_output # type: ignore[return-value]
|
||||
|
||||
|
||||
@given('I have a valid structured output with all required fields')
|
||||
@when('I validate the structured output')
|
||||
def step_validate_structured(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
ctx.struct_result = validate_structured_output(o)
|
||||
|
||||
|
||||
@given('I have a structured output with "{field}" set to {value} (and all other required fields present)')
|
||||
def step_structured_output_field(ctx: Context, field: str, value) -> Any: # type: ignore[no-untyped-def]
|
||||
"""Set a single field on the structured output."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o[field] = _parse_value(str(value))
|
||||
|
||||
|
||||
@given('I have a structured output with empty "elements" list (and all other fields valid)')
|
||||
def step_empty_elements(ctx: Context) -> None:
|
||||
"""Empty elements is valid."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o["elements"] = []
|
||||
|
||||
|
||||
@given('I have a structured output with an element missing "kind" field')
|
||||
def step_invalid_element(ctx: Context) -> None:
|
||||
"""Create an invalid element."""
|
||||
o = _ensure_struct_output(ctx)
|
||||
o["elements"] = [{"not_kind": "panel"}]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Dispatcher scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('I target validation for "{target}" with valid data')
|
||||
def step_dispatcher_target(ctx: Context, target: str) -> None:
|
||||
"""Set up a dispatcher test."""
|
||||
ctx.target_type = target
|
||||
if target in ("plan_tree", "plan-tree", "decisions", "decision-tree", "tree"):
|
||||
ctx.validator_data = [
|
||||
{"decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": 0, "question": "Q", "children": []}
|
||||
]
|
||||
elif target in ("decision", "decision_cli", "decision-cli", "cli_dict", "as_cli_dict"):
|
||||
ctx.validator_data = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Q?",
|
||||
"chosen": "A",
|
||||
"confidence": 0.5,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
elif target in ("structured_output", "structured-output", "session_output", "output_session"):
|
||||
ctx.validator_data = {
|
||||
"command": "test",
|
||||
"session_id": VALID_ULID_A,
|
||||
"status": "ok",
|
||||
"exit_code": 0,
|
||||
"elements": [],
|
||||
}
|
||||
else:
|
||||
ctx.validator_data = {}
|
||||
|
||||
|
||||
@when('I call validate_structured_component_output')
|
||||
def step_call_dispatcher(ctx: Context) -> None:
|
||||
"""Call the dispatcher."""
|
||||
target = ctx.target_type if hasattr(ctx, "target_type") else "plan_tree" # type: ignore[attr-defined]
|
||||
data = ctx.validator_data if hasattr(ctx, "validator_data") else {} # type: ignore[attr-defined]
|
||||
try:
|
||||
ctx.dispatcher_result = validate_structured_component_output(target, data) # type: ignore[arg-type]
|
||||
ctx.dispatcher_error = None
|
||||
except ValidationError as exc:
|
||||
ctx.dispatcher_result = None
|
||||
ctx.dispatcher_error = exc
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Shared scenarios
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@given('the plan tree has {node_count:d} valid nodes')
|
||||
def step_valid_tree(ctx: Context, node_count: int = 3) -> None:
|
||||
"""Set up a default valid tree."""
|
||||
if not hasattr(ctx, "tree_nodes"):
|
||||
ctx.tree_nodes = [
|
||||
{"decision_id": f"01ARZ3NDEKTSV4XXFFJFRC{str(i).zfill(4)}",
|
||||
"type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": []}
|
||||
for i in range(node_count)
|
||||
]
|
||||
|
||||
|
||||
@given('I have a plan tree with valid nodes')
|
||||
def step_valid_nodes_default(ctx: Context) -> None:
|
||||
"""Set default valid tree if no tree defined."""
|
||||
_valid_tree = False # handled by node_count scenario; nothing extra
|
||||
|
||||
|
||||
@when('I validate the plan tree')
|
||||
def step_validate_plan_tree(ctx: Context) -> None:
|
||||
"""Validate and store results."""
|
||||
nodes = ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [ # type: ignore[attr-defined]
|
||||
{"decision_id": VALID_ULID_A, "type": "strategy_choice", "sequence": i, "question": f"Q{i}", "children": []}
|
||||
for i in range(3)
|
||||
]
|
||||
ctx.validation_result = validate_plan_tree(nodes)
|
||||
|
||||
|
||||
@when('I validate the plan tree structure')
|
||||
def step_validate_structure(ctx: Context) -> None:
|
||||
"""Run plan_tree validation."""
|
||||
nodes = ctx.tree_nodes if hasattr(ctx, "tree_nodes") else [] # type: ignore[attr-defined]
|
||||
ctx.validation_result = validate_plan_tree(nodes)
|
||||
|
||||
|
||||
@given('I have a decision dict where whitespace may vary between fields')
|
||||
def step_whitespace_dict(ctx: Context) -> None:
|
||||
"""Whitespace in string values - structure still valid."""
|
||||
ctx.decision_dict = {
|
||||
"decision_id": VALID_ULID_A,
|
||||
"plan_id": VALID_ULID_B,
|
||||
"type": " strategy_choice ",
|
||||
"sequence": 1,
|
||||
"question": "Which approach? ",
|
||||
"chosen": " Option A ",
|
||||
"confidence": 0.75,
|
||||
"parent": "(root)",
|
||||
"is_correction": False,
|
||||
"superseded": False,
|
||||
}
|
||||
|
||||
|
||||
@given('a plan tree with parent and child node references connected by ULID')
|
||||
def step_parent_child_tree(ctx: Context) -> None:
|
||||
"""Parent-decoy pattern - root plus children."""
|
||||
ctx.tree_nodes = [
|
||||
{
|
||||
"decision_id": VALID_ULID_A,
|
||||
"type": "prompt_definition",
|
||||
"sequence": 0,
|
||||
"question": "Root question",
|
||||
"children": [{"decision_id": VALID_ULID_B}, {"decision_id": VALID_ULID_C}],
|
||||
"parent_decision_id": None,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_B,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 1,
|
||||
"question": "Child A",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_A,
|
||||
},
|
||||
{
|
||||
"decision_id": VALID_ULID_C,
|
||||
"type": "strategy_choice",
|
||||
"sequence": 2,
|
||||
"question": "Child B",
|
||||
"children": [],
|
||||
"parent_decision_id": VALID_ULID_A,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@given('I have invalid plan tree data')
|
||||
def step_invalid_tree_data(ctx: Context) -> None:
|
||||
"""Create invalid plan tree node."""
|
||||
ctx.tree_nodes = [
|
||||
{"not_a_key": "value"}, # missing required fields entirely
|
||||
]
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────
|
||||
# Assertions (shared / cross-scenario)
|
||||
# ────────────────────────────────────────────────────────────
|
||||
|
||||
@then('it should be structurally valid')
|
||||
def step_should_be_valid(ctx: Context) -> None:
|
||||
"""Assert validation passed."""
|
||||
result = getattr(ctx, "validation_result", {}) or {"valid": True}
|
||||
assert result["valid"], f"Expected valid but got errors: {result.get('errors', [])}"
|
||||
|
||||
|
||||
@then('it should be structurally invalid')
|
||||
def step_should_be_invalid(ctx: Context) -> None:
|
||||
"""Assert validation failed."""
|
||||
result = getattr(ctx, "validation_result", {}) or {"valid": True}
|
||||
assert not result["valid"], f"Expected invalid but got valid"
|
||||
|
||||
|
||||
@then('it should report no errors')
|
||||
def step_no_errors(ctx: Context) -> None:
|
||||
"""Assert no error messages."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert len(errs) == 0, f"Expected no errors but got: {errs}"
|
||||
|
||||
|
||||
@then('it should report errors including "{text}"')
|
||||
def step_errors_include(ctx: Context, text: str) -> None:
|
||||
"""Assert at least one error contains the given substring."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}"
|
||||
|
||||
|
||||
@then('error message must include "{text}"')
|
||||
@then('error must include "{text}"')
|
||||
def step_error_contains(ctx: Context, text: str) -> None:
|
||||
"""Assert an error contains substring."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert any(text in e for e in errs), f"'{text}' not found in errors: {errs}"
|
||||
|
||||
|
||||
@then('the first error message should identify "{prefix}" and the specific field issue')
|
||||
def step_first_error_identifies(ctx: Context, prefix: str) -> None:
|
||||
"""Assert first error contains prefix."""
|
||||
result = getattr(ctx, "validation_result", {}) or {}
|
||||
errs = result.get("errors", [])
|
||||
assert len(errs) > 0, "Expected at least one error"
|
||||
assert prefix in errs[0], f"First error '{errs[0]}' should contain '{prefix}'"
|
||||
|
||||
|
||||
@then('it should raise a ValidationError with message containing "{text}"')
|
||||
def step_raise_error_with_text(ctx: Context, text: str) -> None:
|
||||
"""Assert dispatcher raised specific error."""
|
||||
err = getattr(ctx, "dispatcher_error", None)
|
||||
assert err is not None, "Expected a ValidationError"
|
||||
assert isinstance(err, ValidationError), f"Expected ValidationError, got {type(err)}"
|
||||
assert text in str(err), f"'{text}' not in error message: {err}"
|
||||
|
||||
|
||||
@then('it should dispatch to the matching validator')
|
||||
@then('return a valid result')
|
||||
def step_dispatcher_valid(ctx: Context) -> None:
|
||||
"""Assert dispatcher succeeded and returned valid."""
|
||||
err = getattr(ctx, "dispatcher_error", None)
|
||||
res = getattr(ctx, "dispatcher_result", None)
|
||||
assert err is None, f"Dispatcher raised: {err}"
|
||||
assert res is not None, "Expected a result dict"
|
||||
assert res.get("valid"), f"Result not valid: {res}"
|
||||
|
||||
|
||||
@then('validation should pass (structure matched, not exact characters)')
|
||||
def step_structure_matched(ctx: Context) -> None:
|
||||
"""Whitespace-tolerant structural validation passes."""
|
||||
d = getattr(ctx, "decision_dict", {})
|
||||
ctx.validation_result = validate_decision_dict(d) # type: ignore[attr-defined]
|
||||
assert ctx.validation_result["valid"], f"Structural validation failed: {ctx.validation_result.get('errors', [])}" # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@then('parent-child relationships should be validly connected')
|
||||
def step_relationships_valid(ctx: Context) -> None:
|
||||
"""Parent-child nodes validate without errors."""
|
||||
result = ctx.validation_result if hasattr(ctx, "validation_result") else {} # type: ignore[attr-defined]
|
||||
assert result.get("valid", False), f"Relation validation failed: {result.get('errors', [])}"
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Steps for tdd_cleanup_stale_destroys_execute_output.feature.
|
||||
|
||||
TDD issue-capture test for bug #11121:
|
||||
_create_sandbox_for_plan() calls GitWorktreeSandbox.cleanup_stale() unconditionally,
|
||||
destroying the cleveragents/plan-<id> branch when the plan is already in
|
||||
execute/complete or execute/processing state (awaiting apply or still in progress).
|
||||
|
||||
The scenarios are tagged @tdd_issue_11121 as permanent regression guards.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
_PLAN_ID = "01TDDSANDBOX000000000000A"
|
||||
_BRANCH_NAME = f"cleveragents/plan-{_PLAN_ID}"
|
||||
|
||||
|
||||
def _git(args: list[str], cwd: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
def _branch_exists(repo_path: str, branch: str) -> bool:
|
||||
"""Return True if the given branch exists in the repo."""
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "--verify", f"refs/heads/{branch}"],
|
||||
cwd=repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _init_git_repo(path: str) -> None:
|
||||
_git(["init", "-q", "-b", "main"], path)
|
||||
_git(["config", "user.name", "TDD Test"], path)
|
||||
_git(["config", "user.email", "tdd@test.local"], path)
|
||||
_git(["config", "commit.gpgsign", "false"], path)
|
||||
|
||||
|
||||
def _build_execute_complete_mocks(
|
||||
context: Context,
|
||||
repo_path: str,
|
||||
) -> None:
|
||||
"""Build mock service + container for a plan in execute/complete state."""
|
||||
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
|
||||
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_type_name = "git-checkout"
|
||||
mock_resource.location = repo_path
|
||||
mock_resource.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_lr = MagicMock()
|
||||
mock_lr.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_project.linked_resources = [mock_lr]
|
||||
|
||||
# Plan is in execute/complete state — this is the critical state for the bug
|
||||
mock_plan = MagicMock()
|
||||
mock_plan.project_links = [MagicMock(project_name="local/tdd-11121-project")]
|
||||
mock_plan.phase = PlanPhase.EXECUTE
|
||||
mock_plan.processing_state = ProcessingState.COMPLETE
|
||||
mock_plan.state = ProcessingState.COMPLETE
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_plan.return_value = mock_plan
|
||||
|
||||
mock_project_repo = MagicMock()
|
||||
mock_project_repo.get.return_value = mock_project
|
||||
|
||||
mock_resource_registry = MagicMock()
|
||||
mock_resource_registry.show_resource.return_value = mock_resource
|
||||
|
||||
mock_container = MagicMock()
|
||||
mock_container.namespaced_project_repo.return_value = mock_project_repo
|
||||
mock_container.resource_registry_service.return_value = mock_resource_registry
|
||||
|
||||
context.tdd11121_service = mock_service
|
||||
context.tdd11121_container = mock_container
|
||||
context.tdd11121_repo_path = repo_path
|
||||
context.tdd11121_plan_id = _PLAN_ID
|
||||
context.tdd11121_branch = _BRANCH_NAME
|
||||
|
||||
|
||||
@given("a temp git repo with an execute-output branch for tdd 11121")
|
||||
def step_create_git_repo_with_execute_output(context: Context) -> None:
|
||||
"""Create a real git repo with a cleveragents/plan-<id> branch holding execute output.
|
||||
|
||||
This simulates the state after a successful plan execute: the worktree branch
|
||||
exists and contains at least one committed file representing execution output.
|
||||
"""
|
||||
d = tempfile.mkdtemp(prefix="tdd-11121-")
|
||||
context.add_cleanup(shutil.rmtree, d, True)
|
||||
|
||||
# Initialise the repo with a base commit on main
|
||||
_init_git_repo(d)
|
||||
Path(d, "README.md").write_text("# project\n")
|
||||
_git(["add", "."], d)
|
||||
_git(["commit", "-q", "-m", "init: base commit"], d)
|
||||
|
||||
# Create the cleveragents/plan-<id> branch and commit an output file to it.
|
||||
# This simulates what _commit_worktree_changes() does after execute completes.
|
||||
_git(["checkout", "-q", "-b", _BRANCH_NAME], d)
|
||||
output_file = Path(d, "generated_output.py")
|
||||
output_file.write_text("# Generated by plan execute\nresult = 42\n")
|
||||
_git(["add", "."], d)
|
||||
_git(["commit", "-q", "-m", f"cleveragents: execute output for plan {_PLAN_ID}"], d)
|
||||
|
||||
# Return to main so the repo is in a normal state
|
||||
_git(["checkout", "-q", "main"], d)
|
||||
|
||||
context.tdd11121_repo_path = d
|
||||
# Verify the branch exists before the test
|
||||
assert _branch_exists(d, _BRANCH_NAME), (
|
||||
f"Pre-condition failed: branch {_BRANCH_NAME} should exist before test"
|
||||
)
|
||||
|
||||
|
||||
def _build_execute_processing_mocks(
|
||||
context: Context,
|
||||
repo_path: str,
|
||||
) -> None:
|
||||
"""Build mock service + container for a plan in execute/processing state."""
|
||||
from cleveragents.domain.models.core.plan import PlanPhase, ProcessingState
|
||||
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_type_name = "git-checkout"
|
||||
mock_resource.location = repo_path
|
||||
mock_resource.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_lr = MagicMock()
|
||||
mock_lr.resource_id = "res-tdd-11121-test"
|
||||
|
||||
mock_project = MagicMock()
|
||||
mock_project.linked_resources = [mock_lr]
|
||||
|
||||
mock_plan = MagicMock()
|
||||
mock_plan.project_links = [MagicMock(project_name="local/tdd-11121-project")]
|
||||
mock_plan.phase = PlanPhase.EXECUTE
|
||||
mock_plan.processing_state = ProcessingState.PROCESSING
|
||||
mock_plan.state = ProcessingState.PROCESSING
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.get_plan.return_value = mock_plan
|
||||
|
||||
mock_project_repo = MagicMock()
|
||||
mock_project_repo.get.return_value = mock_project
|
||||
|
||||
mock_resource_registry = MagicMock()
|
||||
mock_resource_registry.show_resource.return_value = mock_resource
|
||||
|
||||
mock_container = MagicMock()
|
||||
mock_container.namespaced_project_repo.return_value = mock_project_repo
|
||||
mock_container.resource_registry_service.return_value = mock_resource_registry
|
||||
|
||||
context.tdd11121_service = mock_service
|
||||
context.tdd11121_container = mock_container
|
||||
context.tdd11121_repo_path = repo_path
|
||||
context.tdd11121_plan_id = _PLAN_ID
|
||||
context.tdd11121_branch = _BRANCH_NAME
|
||||
|
||||
|
||||
@given("a mocked plan service with the plan in execute/complete state for tdd 11121")
|
||||
def step_mock_service_execute_complete(context: Context) -> None:
|
||||
"""Set up mock service returning a plan in execute/complete state."""
|
||||
_build_execute_complete_mocks(context, context.tdd11121_repo_path)
|
||||
|
||||
|
||||
@given("a mocked plan service with the plan in execute/processing state for tdd 11121")
|
||||
def step_mock_service_execute_processing(context: Context) -> None:
|
||||
"""Set up mock service returning a plan in execute/processing state."""
|
||||
_build_execute_processing_mocks(context, context.tdd11121_repo_path)
|
||||
|
||||
|
||||
@when(
|
||||
"I call _create_sandbox_for_plan a second time on the execute/complete plan for tdd 11121"
|
||||
)
|
||||
def step_call_create_sandbox_second_time(context: Context) -> None:
|
||||
"""Call _create_sandbox_for_plan on a plan already in execute/complete state.
|
||||
|
||||
This simulates the user re-running 'agents plan execute <PLAN_ID>' after
|
||||
execution has already completed. The bug causes cleanup_stale to run and
|
||||
destroy the cleveragents/plan-<id> branch that holds the execute output.
|
||||
"""
|
||||
from cleveragents.cli.commands.plan import _create_sandbox_for_plan
|
||||
|
||||
context.tdd11121_second_call_exception: Exception | None = None
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.application.container.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
):
|
||||
_sandbox_root, sandbox_infos = _create_sandbox_for_plan(
|
||||
context.tdd11121_plan_id,
|
||||
context.tdd11121_service,
|
||||
)
|
||||
context.tdd11121_sandbox_infos = sandbox_infos
|
||||
# Clean up any newly created sandbox to avoid resource leaks
|
||||
for sinfo in sandbox_infos:
|
||||
with contextlib.suppress(Exception):
|
||||
sinfo.sandbox_obj.cleanup()
|
||||
except Exception as exc:
|
||||
# Record but don't re-raise — we want to check branch state regardless
|
||||
context.tdd11121_second_call_exception = exc
|
||||
|
||||
|
||||
@when(
|
||||
"I call _create_sandbox_for_plan a second time on the execute/processing plan for tdd 11121"
|
||||
)
|
||||
def step_call_create_sandbox_second_time_processing(context: Context) -> None:
|
||||
"""Call _create_sandbox_for_plan on a plan already in execute/processing state.
|
||||
|
||||
This simulates the user re-running 'agents plan execute <PLAN_ID>' while the
|
||||
plan is still in progress (execute/processing). The guard must protect the
|
||||
sandbox branch from cleanup_stale in this state too.
|
||||
"""
|
||||
from cleveragents.cli.commands.plan import _create_sandbox_for_plan
|
||||
|
||||
context.tdd11121_second_call_exception: Exception | None = None
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.application.container.get_container",
|
||||
return_value=context.tdd11121_container,
|
||||
),
|
||||
):
|
||||
_sandbox_root, sandbox_infos = _create_sandbox_for_plan(
|
||||
context.tdd11121_plan_id,
|
||||
context.tdd11121_service,
|
||||
)
|
||||
context.tdd11121_sandbox_infos = sandbox_infos
|
||||
for sinfo in sandbox_infos:
|
||||
with contextlib.suppress(Exception):
|
||||
sinfo.sandbox_obj.cleanup()
|
||||
except Exception as exc:
|
||||
context.tdd11121_second_call_exception = exc
|
||||
|
||||
|
||||
@then(
|
||||
"the cleveragents plan branch should still exist after the second call for tdd 11121"
|
||||
)
|
||||
def step_branch_still_exists(context: Context) -> None:
|
||||
"""Assert the cleveragents/plan-<id> branch was NOT destroyed by cleanup_stale.
|
||||
|
||||
The guard in _create_sandbox_for_plan skips cleanup_stale when the plan is
|
||||
already in execute/processing or execute/complete state, preserving the
|
||||
sandbox branch so plan apply can merge it.
|
||||
"""
|
||||
repo_path = context.tdd11121_repo_path
|
||||
branch = context.tdd11121_branch
|
||||
assert _branch_exists(repo_path, branch), (
|
||||
f"Bug #11121: branch '{branch}' was destroyed by cleanup_stale during "
|
||||
f"a second _create_sandbox_for_plan call. "
|
||||
f"The branch must survive until plan apply merges it."
|
||||
)
|
||||
|
||||
|
||||
@then("the apply sandbox changes should find at least one artifact for tdd 11121")
|
||||
def step_apply_finds_artifacts(context: Context) -> None:
|
||||
"""Assert that apply can find the execute output after a re-invoked execute.
|
||||
|
||||
The guard in _create_sandbox_for_plan preserves the sandbox branch, so the
|
||||
git diff between HEAD and the plan branch finds at least one artifact.
|
||||
"""
|
||||
repo_path = context.tdd11121_repo_path
|
||||
branch = context.tdd11121_branch
|
||||
|
||||
if not _branch_exists(repo_path, branch):
|
||||
raise AssertionError(
|
||||
f"Bug #11121: branch '{branch}' was destroyed by cleanup_stale. "
|
||||
f"plan apply would find 0 artifacts (empty changeset). "
|
||||
f"The branch must persist until apply merges it."
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--stat", f"HEAD...{branch}"],
|
||||
cwd=repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
stat_lines = (result.stdout or "").strip().splitlines()
|
||||
artifact_count = max(0, len(stat_lines) - 1) if stat_lines else 0
|
||||
|
||||
assert artifact_count > 0, (
|
||||
f"Bug #11121: plan apply would find {artifact_count} artifacts after "
|
||||
f"re-invoked execute on execute/complete plan. Expected >= 1 artifact. "
|
||||
f"diff --stat output: {result.stdout!r}"
|
||||
)
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Step definitions for TDD Issue #9131 — invariant_enforced decisions not
|
||||
propagated to child plans on subplan spawn.
|
||||
|
||||
These steps exercise ``SubplanService.spawn()`` and verify that it propagates
|
||||
``invariant_enforced`` decisions from the parent plan to each child plan's
|
||||
decision tree.
|
||||
|
||||
On ``master`` (before the fix), ``spawn()`` creates child Plan objects but
|
||||
does NOT record ``invariant_enforced`` decisions on them. The assertions in
|
||||
these steps will **fail** until the bug is fixed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.application.services.decision_service import DecisionService
|
||||
from cleveragents.application.services.subplan_service import (
|
||||
SpawnEntry,
|
||||
SpawnResult,
|
||||
SubplanService,
|
||||
)
|
||||
from cleveragents.domain.models.core.decision import (
|
||||
ContextSnapshot,
|
||||
Decision,
|
||||
DecisionType,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
ExecutionMode,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
SubplanConfig,
|
||||
)
|
||||
|
||||
_PARENT_PLAN_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6PN00"
|
||||
_ROOT_PLAN_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6RF00"
|
||||
_SPAWN_DEC_ID: str = "01HGZ6FE0AQDYTR4BXVQZ6DA00"
|
||||
|
||||
|
||||
def _make_parent_plan() -> Plan:
|
||||
"""Create a minimal parent plan for testing."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
root_plan_id=_ROOT_PLAN_ID,
|
||||
),
|
||||
namespaced_name=NamespacedName(namespace="local", name="parent-plan"),
|
||||
description="Parent plan for invariant propagation test",
|
||||
action_name="local/parent-action",
|
||||
)
|
||||
|
||||
|
||||
def _make_spawn_decision(
|
||||
plan_id: str = _PARENT_PLAN_ID,
|
||||
) -> Decision:
|
||||
"""Create a subplan_spawn Decision."""
|
||||
return Decision(
|
||||
decision_id=_SPAWN_DEC_ID,
|
||||
plan_id=plan_id,
|
||||
decision_type=DecisionType.SUBPLAN_SPAWN,
|
||||
sequence_number=10,
|
||||
question="Should we spawn a child plan?",
|
||||
chosen_option="local/sub-action",
|
||||
context_snapshot=ContextSnapshot(),
|
||||
)
|
||||
|
||||
|
||||
def _make_spawn_entry(plan_id: str = _PARENT_PLAN_ID) -> SpawnEntry:
|
||||
"""Create a single spawn entry."""
|
||||
return SpawnEntry(
|
||||
decision=_make_spawn_decision(plan_id),
|
||||
action_name="local/sub-action",
|
||||
description="Child plan for invariant propagation test",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a parent plan with invariant_enforced decisions recorded")
|
||||
def step_parent_with_invariant_decisions(context: Context) -> None:
|
||||
"""Set up a parent plan with one invariant_enforced decision."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
# Record one invariant_enforced decision on the parent plan
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question="Is this invariant enforced?",
|
||||
chosen_option="invariant enforced: do not modify production data",
|
||||
)
|
||||
context.parent_invariant_decisions = [inv_dec]
|
||||
context.expected_invariant_count = 1
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with {count:d} invariant_enforced decisions recorded")
|
||||
def step_parent_with_n_invariant_decisions(context: Context, count: int) -> None:
|
||||
"""Set up a parent plan with N invariant_enforced decisions."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
inv_decisions: list[Decision] = []
|
||||
for i in range(count):
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question=f"Is invariant {i} enforced?",
|
||||
chosen_option=f"invariant enforced: constraint {i}",
|
||||
)
|
||||
inv_decisions.append(inv_dec)
|
||||
|
||||
context.parent_invariant_decisions = inv_decisions
|
||||
context.expected_invariant_count = count
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with a non_overridable invariant_enforced decision")
|
||||
def step_parent_with_non_overridable_invariant(context: Context) -> None:
|
||||
"""Set up a parent plan with a non_overridable invariant_enforced decision."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
|
||||
# Record a non_overridable invariant decision
|
||||
inv_dec = context.decision_service.record_decision(
|
||||
plan_id=_PARENT_PLAN_ID,
|
||||
decision_type=DecisionType.INVARIANT_ENFORCED,
|
||||
question="Is this non-overridable global invariant enforced?",
|
||||
chosen_option="non_overridable invariant enforced: never delete user data",
|
||||
rationale="Global non-overridable invariant must propagate to all child plans",
|
||||
)
|
||||
context.parent_invariant_decisions = [inv_dec]
|
||||
context.non_overridable_chosen_option = inv_dec.chosen_option
|
||||
context.expected_invariant_count = 1
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a parent plan with no invariant_enforced decisions")
|
||||
def step_parent_with_no_invariant_decisions(context: Context) -> None:
|
||||
"""Set up a parent plan with no invariant_enforced decisions."""
|
||||
context.parent_plan = _make_parent_plan()
|
||||
context.decision_service = DecisionService()
|
||||
context.parent_invariant_decisions = []
|
||||
context.expected_invariant_count = 0
|
||||
|
||||
context.subplan_service = SubplanService(
|
||||
decision_service=context.decision_service,
|
||||
)
|
||||
context.subplan_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
)
|
||||
|
||||
|
||||
@given("a valid spawn entry for the parent plan")
|
||||
def step_valid_spawn_entry(context: Context) -> None:
|
||||
"""Create a single valid spawn entry."""
|
||||
context.spawn_entries = [_make_spawn_entry()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I spawn a child plan via SubplanService")
|
||||
def step_spawn_child_plan(context: Context) -> None:
|
||||
"""Call SubplanService.spawn() and capture the result."""
|
||||
context.spawn_error = None
|
||||
try:
|
||||
context.spawn_result = context.subplan_service.spawn(
|
||||
parent_plan=context.parent_plan,
|
||||
config=context.subplan_config,
|
||||
spawn_entries=context.spawn_entries,
|
||||
)
|
||||
except Exception as exc:
|
||||
context.spawn_error = exc
|
||||
context.spawn_result = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then(
|
||||
"the decision service should have recorded invariant_enforced decisions"
|
||||
" for the child plan"
|
||||
)
|
||||
def step_child_has_invariant_decisions(context: Context) -> None:
|
||||
"""Assert that invariant_enforced decisions were recorded for the child plan.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions to
|
||||
child plans. This assertion will fail until the bug is fixed.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
# Query the decision service for invariant_enforced decisions on the child plan
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) > 0, (
|
||||
f"Child plan {child_plan_id!r} has no invariant_enforced decisions. "
|
||||
f"Bug #9131: SubplanService.spawn() does not propagate "
|
||||
f"invariant_enforced decisions from parent plan to child plans."
|
||||
)
|
||||
|
||||
|
||||
@then("the child plan should have {count:d} invariant_enforced decisions recorded")
|
||||
def step_child_has_n_invariant_decisions(context: Context, count: int) -> None:
|
||||
"""Assert that the child plan has exactly N invariant_enforced decisions.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) == count, (
|
||||
f"Child plan {child_plan_id!r} has {len(child_invariant_decisions)} "
|
||||
f"invariant_enforced decisions, expected {count}. "
|
||||
f"Bug #9131: SubplanService.spawn() does not propagate "
|
||||
f"invariant_enforced decisions from parent plan to child plans."
|
||||
)
|
||||
|
||||
|
||||
@then("the child plan should have the non_overridable invariant decision propagated")
|
||||
def step_child_has_non_overridable_invariant(context: Context) -> None:
|
||||
"""Assert that the non_overridable invariant decision was propagated.
|
||||
|
||||
Bug #9131: spawn() does not propagate invariant_enforced decisions.
|
||||
"""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) > 0, (
|
||||
f"Child plan {child_plan_id!r} has no invariant_enforced decisions. "
|
||||
f"Bug #9131: non_overridable invariant was not propagated."
|
||||
)
|
||||
|
||||
# Verify the non_overridable invariant text was propagated
|
||||
expected_option: str = context.non_overridable_chosen_option
|
||||
propagated_options: list[str] = [d.chosen_option for d in child_invariant_decisions]
|
||||
assert expected_option in propagated_options, (
|
||||
f"Non-overridable invariant chosen_option {expected_option!r} not found "
|
||||
f"in child plan decisions: {propagated_options}. "
|
||||
f"Bug #9131: non_overridable invariant was not propagated."
|
||||
)
|
||||
|
||||
|
||||
@then("the spawn should succeed with no invariant decisions recorded for the child")
|
||||
def step_spawn_succeeds_no_invariants(context: Context) -> None:
|
||||
"""Assert that spawn succeeds cleanly when parent has no invariant decisions."""
|
||||
assert context.spawn_error is None, (
|
||||
f"Spawn raised an unexpected error: {context.spawn_error}"
|
||||
)
|
||||
result: SpawnResult = context.spawn_result
|
||||
assert result is not None, "Spawn result is None"
|
||||
assert result.total_spawned == 1, (
|
||||
f"Expected 1 spawned child, got {result.total_spawned}"
|
||||
)
|
||||
|
||||
# When parent has no invariant decisions, child should also have none
|
||||
assert len(result.child_plans) > 0, "No child plans were created"
|
||||
child_plan: Plan = result.child_plans[0]
|
||||
child_plan_id: str = child_plan.identity.plan_id
|
||||
|
||||
child_invariant_decisions: list[Decision] = context.decision_service.list_by_type(
|
||||
child_plan_id,
|
||||
DecisionType.INVARIANT_ENFORCED,
|
||||
)
|
||||
|
||||
assert len(child_invariant_decisions) == 0, (
|
||||
f"Child plan {child_plan_id!r} has {len(child_invariant_decisions)} "
|
||||
f"invariant_enforced decisions, expected 0 (parent had none)."
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Step definitions for TDD issue #7501 — PlanResult.success derivation.
|
||||
|
||||
Validates that PlanRepository._to_domain derives PlanResult.success from
|
||||
the dedicated result_success column rather than from error_message is None.
|
||||
|
||||
Targets ``PlanRepository`` in
|
||||
``src/cleveragents/infrastructure/database/repositories.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from cleveragents.infrastructure.database.models import Base, PlanModel
|
||||
from cleveragents.infrastructure.database.repositories import PlanRepository
|
||||
|
||||
|
||||
def _make_project(session: object) -> int:
|
||||
"""Insert a minimal project row and return its id."""
|
||||
from cleveragents.infrastructure.database.models import ProjectModel
|
||||
|
||||
project = ProjectModel(
|
||||
name="test-project-7501",
|
||||
path="/tmp/test-project-7501",
|
||||
settings={},
|
||||
)
|
||||
session.add(project) # type: ignore[attr-defined]
|
||||
session.flush() # type: ignore[attr-defined]
|
||||
return int(project.id) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a clean in-memory database for plan result success tests")
|
||||
def step_clean_db_result_success(context: Context) -> None:
|
||||
"""Create a fresh in-memory SQLite database with all tables."""
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
context.rs_db_engine = engine
|
||||
session = sessionmaker(bind=engine)()
|
||||
context.rs_db_session = session
|
||||
|
||||
|
||||
@given("a legacy plan repository backed by the database")
|
||||
def step_legacy_plan_repo(context: Context) -> None:
|
||||
"""Instantiate a PlanRepository using the in-memory session."""
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
context.rs_plan_repo = PlanRepository(session=context.rs_db_session)
|
||||
context.rs_retrieved_plan = None
|
||||
context.rs_project_id = _make_project(context.rs_db_session)
|
||||
context.rs_db_session.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _insert_plan_model(
|
||||
session: object,
|
||||
project_id: int,
|
||||
applied_at: datetime | None = None,
|
||||
error_message: str | None = None,
|
||||
result_success: bool | None = None,
|
||||
) -> int:
|
||||
"""Insert a PlanModel row directly and return its id."""
|
||||
db_plan = PlanModel(
|
||||
project_id=project_id,
|
||||
name="test-plan-7501",
|
||||
prompt="Test prompt for issue 7501",
|
||||
status="pending",
|
||||
current=False,
|
||||
applied_at=applied_at,
|
||||
error_message=error_message,
|
||||
result_success=result_success,
|
||||
)
|
||||
session.add(db_plan) # type: ignore[attr-defined]
|
||||
session.flush() # type: ignore[attr-defined]
|
||||
session.commit() # type: ignore[attr-defined]
|
||||
return int(db_plan.id) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a legacy plan with applied_at set and result_success column True")
|
||||
def step_plan_result_success_true(context: Context) -> None:
|
||||
"""Insert a plan row with result_success=True."""
|
||||
context.rs_plan_id = _insert_plan_model(
|
||||
session=context.rs_db_session,
|
||||
project_id=context.rs_project_id,
|
||||
applied_at=datetime.now(),
|
||||
error_message=None,
|
||||
result_success=True,
|
||||
)
|
||||
|
||||
|
||||
@given("a legacy plan with applied_at set and result_success column False")
|
||||
def step_plan_result_success_false(context: Context) -> None:
|
||||
"""Insert a plan row with result_success=False."""
|
||||
context.rs_plan_id = _insert_plan_model(
|
||||
session=context.rs_db_session,
|
||||
project_id=context.rs_project_id,
|
||||
applied_at=datetime.now(),
|
||||
error_message=None,
|
||||
result_success=False,
|
||||
)
|
||||
|
||||
|
||||
@given("a legacy plan with a build error_message and result_success column True")
|
||||
def step_plan_build_error_result_success_true(context: Context) -> None:
|
||||
"""Insert a plan row with a build error but result_success=True.
|
||||
|
||||
This is the core bug scenario: a plan that had a build error but later
|
||||
succeeded in the apply phase. Without the fix, success would be derived
|
||||
as False (because error_message is not None). With the fix, success is
|
||||
correctly derived as True from result_success.
|
||||
"""
|
||||
context.rs_plan_id = _insert_plan_model(
|
||||
session=context.rs_db_session,
|
||||
project_id=context.rs_project_id,
|
||||
applied_at=datetime.now(),
|
||||
error_message="Build phase error: compilation failed",
|
||||
result_success=True,
|
||||
)
|
||||
|
||||
|
||||
@given(
|
||||
"a legacy plan with applied_at set and result_success column NULL and no error_message"
|
||||
)
|
||||
def step_plan_null_result_success_no_error(context: Context) -> None:
|
||||
"""Insert a plan row with result_success=NULL and no error_message.
|
||||
|
||||
Simulates a pre-migration record. The legacy heuristic should mark it
|
||||
as success=True because error_message is None.
|
||||
"""
|
||||
context.rs_plan_id = _insert_plan_model(
|
||||
session=context.rs_db_session,
|
||||
project_id=context.rs_project_id,
|
||||
applied_at=datetime.now(),
|
||||
error_message=None,
|
||||
result_success=None,
|
||||
)
|
||||
|
||||
|
||||
@given(
|
||||
"a legacy plan with applied_at set and result_success column NULL and an error_message"
|
||||
)
|
||||
def step_plan_null_result_success_with_error(context: Context) -> None:
|
||||
"""Insert a plan row with result_success=NULL and an error_message.
|
||||
|
||||
Simulates a pre-migration record that failed. The legacy heuristic
|
||||
should mark it as success=False because error_message is not None.
|
||||
"""
|
||||
context.rs_plan_id = _insert_plan_model(
|
||||
session=context.rs_db_session,
|
||||
project_id=context.rs_project_id,
|
||||
applied_at=datetime.now(),
|
||||
error_message="Apply phase error: deployment failed",
|
||||
result_success=None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("the legacy plan is retrieved from the repository")
|
||||
def step_retrieve_legacy_plan(context: Context) -> None:
|
||||
"""Retrieve the plan via PlanRepository.get_by_id."""
|
||||
context.rs_retrieved_plan = context.rs_plan_repo.get_by_id(context.rs_plan_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the retrieved plan result success should be True")
|
||||
def step_check_result_success_true(context: Context) -> None:
|
||||
"""Assert that the retrieved plan's result.success is True."""
|
||||
plan = context.rs_retrieved_plan
|
||||
assert plan is not None, "Expected a plan but got None"
|
||||
assert plan.result is not None, "Expected plan.result to be set but it was None"
|
||||
assert plan.result.success is True, (
|
||||
f"Expected plan.result.success=True but got {plan.result.success!r}"
|
||||
)
|
||||
|
||||
|
||||
@then("the retrieved plan result success should be False")
|
||||
def step_check_result_success_false(context: Context) -> None:
|
||||
"""Assert that the retrieved plan's result.success is False."""
|
||||
plan = context.rs_retrieved_plan
|
||||
assert plan is not None, "Expected a plan but got None"
|
||||
assert plan.result is not None, "Expected plan.result to be set but it was None"
|
||||
assert plan.result.success is False, (
|
||||
f"Expected plan.result.success=False but got {plan.result.success!r}"
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Step definitions for tdd_session_create_suppress_exception.feature.
|
||||
|
||||
This test captures bug #10414: ``session create`` in
|
||||
``src/cleveragents/cli/commands/session.py`` used
|
||||
``contextlib.suppress(Exception)`` to silently discard ALL exceptions raised
|
||||
by ``_facade_dispatch()`` without any logging. Programming errors,
|
||||
unexpected failures, and infrastructure issues in the facade dispatch were
|
||||
completely invisible.
|
||||
|
||||
The fix replaces the suppress block with a ``try/except Exception`` that
|
||||
calls ``_log.warning(..., exc_info=True)`` so that the exception is
|
||||
recorded but the session creation remains non-fatal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands import session as session_mod
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.domain.models.core.session import Session
|
||||
|
||||
# A distinctive error message to confirm the right exception was raised.
|
||||
_DISPATCH_ERROR_MESSAGE = "facade dispatch failed for tdd_10414"
|
||||
|
||||
|
||||
def _make_mock_session() -> MagicMock:
|
||||
"""Build a minimal mock Session object sufficient for the create command."""
|
||||
mock_session = MagicMock(spec=Session)
|
||||
mock_session.session_id = "01JTEST000000000000000000000"
|
||||
mock_session.actor_name = None
|
||||
mock_session.namespace = "default"
|
||||
mock_session.message_count = 0
|
||||
mock_session.created_at = datetime(2026, 1, 1, 0, 0, 0)
|
||||
mock_session.updated_at = datetime(2026, 1, 1, 0, 0, 0)
|
||||
return mock_session
|
||||
|
||||
|
||||
@given("a session create command with a mocked session service")
|
||||
def step_given_mocked_session_service(context: Context) -> None:
|
||||
"""Set up a CLI runner with a mocked session service.
|
||||
|
||||
The mock service returns a valid session so that the create command
|
||||
proceeds past the service call and reaches the ``_facade_dispatch``
|
||||
try/except block.
|
||||
"""
|
||||
context.runner = CliRunner()
|
||||
|
||||
mock_service = MagicMock()
|
||||
mock_service.create.return_value = _make_mock_session()
|
||||
|
||||
# Patch the module-level service so the CLI uses our mock.
|
||||
context._orig_service = session_mod._service
|
||||
session_mod._service = mock_service
|
||||
|
||||
def _cleanup() -> None:
|
||||
session_mod._service = context._orig_service
|
||||
|
||||
context.add_cleanup(_cleanup)
|
||||
|
||||
|
||||
@given("the facade dispatch is patched to raise a RuntimeError")
|
||||
def step_given_facade_dispatch_raises(context: Context) -> None:
|
||||
"""Patch ``_facade_dispatch`` so it always raises a RuntimeError.
|
||||
|
||||
This simulates a real failure in the facade layer (e.g. the A2A
|
||||
bootstrap is unavailable) and exercises the ``try/except`` block in
|
||||
the ``create`` command.
|
||||
"""
|
||||
context._facade_patcher = patch(
|
||||
"cleveragents.cli.commands.session._facade_dispatch",
|
||||
side_effect=RuntimeError(_DISPATCH_ERROR_MESSAGE),
|
||||
)
|
||||
context._facade_patcher.start()
|
||||
|
||||
def _cleanup() -> None:
|
||||
context._facade_patcher.stop()
|
||||
|
||||
context.add_cleanup(_cleanup)
|
||||
|
||||
|
||||
@when("I invoke the session create command via the CLI runner")
|
||||
def step_when_invoke_create(context: Context) -> None:
|
||||
"""Invoke ``session create`` and capture log records during the call."""
|
||||
# Capture WARNING-level log records from the session module logger.
|
||||
session_logger = logging.getLogger("cleveragents.cli.commands.session")
|
||||
handler = _CapturingHandler()
|
||||
handler.setLevel(logging.WARNING)
|
||||
session_logger.addHandler(handler)
|
||||
session_logger.setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
context.result = context.runner.invoke(session_app, ["create"])
|
||||
finally:
|
||||
session_logger.removeHandler(handler)
|
||||
|
||||
context.captured_log_records = handler.records
|
||||
|
||||
|
||||
class _CapturingHandler(logging.Handler):
|
||||
"""A logging handler that stores all emitted records in a list."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.records: list[logging.LogRecord] = []
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
self.records.append(record)
|
||||
|
||||
|
||||
@then("a WARNING log entry should have been emitted for the facade dispatch failure")
|
||||
def step_then_warning_logged(context: Context) -> None:
|
||||
"""Assert that a WARNING log entry was emitted when the facade dispatch raised.
|
||||
|
||||
The fix replaces ``contextlib.suppress(Exception)`` with a
|
||||
``try/except Exception`` block that calls ``_log.warning(..., exc_info=True)``.
|
||||
This assertion verifies the fix is in place.
|
||||
"""
|
||||
# The session create command should still succeed (non-fatal).
|
||||
assert context.result.exit_code == 0, (
|
||||
f"Expected session create to exit 0 even when facade dispatch fails, "
|
||||
f"but got exit code {context.result.exit_code}.\n"
|
||||
f"Output:\n{context.result.output}"
|
||||
)
|
||||
|
||||
# Assert that a WARNING log record was emitted.
|
||||
warning_records = [
|
||||
r for r in context.captured_log_records if r.levelno >= logging.WARNING
|
||||
]
|
||||
assert warning_records, (
|
||||
f"Bug #10414: No WARNING log record was emitted when _facade_dispatch() "
|
||||
f"raised a RuntimeError inside the try/except block. "
|
||||
f"Captured records: {context.captured_log_records}"
|
||||
)
|
||||
|
||||
# Assert the log record references the facade dispatch failure.
|
||||
all_messages = " ".join(r.getMessage() for r in warning_records)
|
||||
assert _DISPATCH_ERROR_MESSAGE in all_messages or any(
|
||||
r.exc_info is not None for r in warning_records
|
||||
), (
|
||||
f"Bug #10414: WARNING log record was found but did not contain the "
|
||||
f"expected error message {_DISPATCH_ERROR_MESSAGE!r} or exc_info. "
|
||||
f"Log messages: {all_messages!r}"
|
||||
)
|
||||
@@ -15,18 +15,21 @@ from unittest.mock import MagicMock, patch
|
||||
from behave import given, then, when
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.application.services.session_workflow import TellResult
|
||||
from cleveragents.cli.commands.session import app as session_app
|
||||
from cleveragents.domain.models.core.session import SessionService
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
_ULID = "01KJ1N8YDN05N29P5RZV4SPDVZ"
|
||||
_STUB_RESPONSE = "Acknowledged: Hello world"
|
||||
|
||||
|
||||
def _mock_service() -> MagicMock:
|
||||
"""Create a MagicMock that passes isinstance checks for SessionService."""
|
||||
svc = MagicMock(spec=SessionService)
|
||||
svc.append_message.return_value = None
|
||||
svc.get_messages.return_value = []
|
||||
return svc
|
||||
|
||||
|
||||
@@ -37,6 +40,33 @@ def _patch_service(context: object, svc: MagicMock) -> None:
|
||||
context._cleanup_handlers.append(patcher.stop) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def _mock_workflow(response: str = _STUB_RESPONSE) -> MagicMock:
|
||||
"""Create a mock SessionWorkflow with canned tell and tell_stream returns."""
|
||||
wf = MagicMock()
|
||||
wf.tell.return_value = TellResult(
|
||||
session_id=_ULID,
|
||||
user_message="",
|
||||
assistant_message=response,
|
||||
input_tokens=5,
|
||||
output_tokens=5,
|
||||
cost=0.0,
|
||||
duration_ms=0.0,
|
||||
tool_calls_count=0,
|
||||
)
|
||||
wf.tell_stream.return_value = iter([response])
|
||||
return wf
|
||||
|
||||
|
||||
def _patch_workflow(context: object, wf: MagicMock) -> None:
|
||||
"""Patch _build_session_workflow and register cleanup."""
|
||||
patcher = patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
return_value=wf,
|
||||
)
|
||||
patcher.start()
|
||||
context._cleanup_handlers.append(patcher.stop) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -45,8 +75,11 @@ def _patch_service(context: object, svc: MagicMock) -> None:
|
||||
@given("a session tell stream redaction mock service")
|
||||
def step_setup_mock_service(context: object) -> None:
|
||||
svc = _mock_service()
|
||||
wf = _mock_workflow()
|
||||
_patch_service(context, svc)
|
||||
_patch_workflow(context, wf)
|
||||
context.mock_svc = svc # type: ignore[attr-defined]
|
||||
context.mock_wf = wf # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -117,10 +150,15 @@ def step_stdout_write_not_called(context: object) -> None:
|
||||
return original_write(s)
|
||||
|
||||
svc = _mock_service()
|
||||
wf = _mock_workflow()
|
||||
sensitive_prompt = "my-api-key=sk-secret1234567890abcdef"
|
||||
|
||||
with (
|
||||
mock_patch("cleveragents.cli.commands.session._service", svc),
|
||||
mock_patch(
|
||||
"cleveragents.cli.commands.session._build_session_workflow",
|
||||
return_value=wf,
|
||||
),
|
||||
mock_patch("sys.stdout.write", side_effect=tracking_write),
|
||||
):
|
||||
runner.invoke(
|
||||
@@ -159,8 +197,8 @@ def step_output_via_console(context: object) -> None:
|
||||
that it was NOT split into individual characters in the captured output.
|
||||
"""
|
||||
result = context.result # type: ignore[attr-defined]
|
||||
# The assistant content for "Hello world" is "Acknowledged: Hello world"
|
||||
expected = "Acknowledged: Hello world"
|
||||
# After real LLM wiring, the stub workflow returns _STUB_RESPONSE.
|
||||
expected = _STUB_RESPONSE
|
||||
assert expected in result.output, (
|
||||
f"Expected '{expected}' in streaming output.\nGot:\n{result.output}"
|
||||
)
|
||||
|
||||
@@ -82,20 +82,20 @@ def _build_mock_textual():
|
||||
def update(self, text):
|
||||
self._text = text
|
||||
|
||||
class MockTextArea:
|
||||
"""Minimal TextArea stand-in for the Textual base class."""
|
||||
class MockInput:
|
||||
"""Minimal Input stand-in for the Textual base class."""
|
||||
|
||||
text = ""
|
||||
value = ""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.text = ""
|
||||
self.value = ""
|
||||
|
||||
mock_textual_app.App = MockApp
|
||||
mock_textual_containers.Vertical = MockVertical
|
||||
mock_textual_widgets.Header = MockHeader
|
||||
mock_textual_widgets.Footer = MockFooter
|
||||
mock_textual_widgets.Static = MockStatic
|
||||
mock_textual_widgets.TextArea = MockTextArea
|
||||
mock_textual_widgets.Input = MockInput
|
||||
|
||||
return {
|
||||
"textual": mock_textual,
|
||||
@@ -114,7 +114,7 @@ def _install_mock_textual(context):
|
||||
for key, mod in mocks.items():
|
||||
sys.modules[key] = mod
|
||||
|
||||
# Reload widget modules so they pick up the mock Static/TextArea base class
|
||||
# Reload widget modules so they pick up the mock Static/Input base class
|
||||
import cleveragents.tui.widgets.help_panel_overlay as hp_mod
|
||||
import cleveragents.tui.widgets.persona_bar as pb_mod
|
||||
import cleveragents.tui.widgets.prompt as prompt_mod
|
||||
@@ -142,7 +142,7 @@ def _restore_modules(context):
|
||||
else:
|
||||
sys.modules[key] = val
|
||||
|
||||
# Reload widget modules so they pick up the real Static/TextArea base class again
|
||||
# Reload widget modules so they pick up the real Static/Input base class again
|
||||
import cleveragents.tui.widgets.help_panel_overlay as hp_mod
|
||||
import cleveragents.tui.widgets.persona_bar as pb_mod
|
||||
import cleveragents.tui.widgets.prompt as prompt_mod
|
||||
@@ -370,7 +370,7 @@ def step_set_prompt_text(context, text):
|
||||
from cleveragents.tui.widgets.prompt import PromptInput
|
||||
|
||||
prompt = context._tui_app.query_one("#prompt", PromptInput)
|
||||
prompt.text = text
|
||||
prompt.value = text
|
||||
|
||||
|
||||
@then('the conversation widget should contain "{text}"')
|
||||
@@ -425,11 +425,11 @@ def step_persona_bar_shows_name(context):
|
||||
# on_input_submitted helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _submit_text(context, text):
|
||||
"""Set prompt text and fire on_input_submitted."""
|
||||
"""Set prompt value and fire on_input_submitted."""
|
||||
from cleveragents.tui.widgets.prompt import PromptInput
|
||||
|
||||
prompt = context._tui_app.query_one("#prompt", PromptInput)
|
||||
prompt.text = text
|
||||
prompt.value = text
|
||||
event = SimpleNamespace()
|
||||
context._tui_app.on_input_submitted(event)
|
||||
|
||||
|
||||
@@ -143,13 +143,3 @@ def step_run_fallback_tui_app(context: Context) -> None:
|
||||
def step_fallback_tui_fails(context: Context, message: str) -> None:
|
||||
assert context.tui_fallback_error is not None
|
||||
assert message in str(context.tui_fallback_error)
|
||||
|
||||
|
||||
@when('I detect mode for "{text}"')
|
||||
def step_detect_mode(context: Context, text: str) -> None:
|
||||
context.detected_mode = InputModeRouter.detect_mode(text)
|
||||
|
||||
|
||||
@then('the detected mode should be "{mode}"')
|
||||
def step_detected_mode_equals(context: Context, mode: str) -> None:
|
||||
assert context.detected_mode.value == mode
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user