test(e2e): add M6 autonomy acceptance suite #470
@@ -2,6 +2,14 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Added comprehensive M6 autonomy acceptance test suite covering ACP local facade dispatch
|
||||
(session/plan/registry/context/event operations), event queue pub/sub with local callbacks
|
||||
and close semantics, HTTP transport stub rejection, version negotiation, automation profile
|
||||
built-ins (8 profiles), custom profile creation/validation/YAML loading, guard enforcement
|
||||
(denylist, allowlist, call-limit, cost-budget, write-approval, apply-approval), and profile
|
||||
service 4-level resolution precedence. Includes Behave BDD scenarios (52), Robot Framework
|
||||
integration tests (11), ASV performance benchmarks (5 suites), JSON fixtures, and
|
||||
documentation update. (#211)
|
||||
- Added MCP refresh hooks to wire `notifications/tools/list_changed` events from MCP servers
|
||||
to `SkillRegistry`. Introduced `SkillRegistry.refresh(name)` and `refresh_all()` to
|
||||
recompute flattened tool sets on demand. Added `MCPRefreshHook` with configurable debounce
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""ASV benchmarks for M6 autonomy acceptance suite runtime.
|
||||
|
||||
Measures the performance of:
|
||||
- ACP local facade dispatch operations
|
||||
- Automation guard evaluation
|
||||
- AutomationProfileService resolution precedence
|
||||
- ACP event queue publish/subscribe
|
||||
- Fixture loading overhead
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.acp.events import AcpEventQueue # noqa: E402
|
||||
from cleveragents.acp.facade import AcpLocalFacade # noqa: E402
|
||||
from cleveragents.acp.models import AcpEvent, AcpRequest # noqa: E402
|
||||
from cleveragents.acp.versioning import AcpVersionNegotiator # noqa: E402
|
||||
from cleveragents.application.services.automation_profile_service import ( # noqa: E402
|
||||
AutomationProfileService,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_guard import ( # noqa: E402
|
||||
AutomationGuard,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import ( # noqa: E402
|
||||
AutomationProfile,
|
||||
)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m6"
|
||||
|
||||
|
||||
class M6FacadeDispatchSuite:
|
||||
"""Benchmark ACP local facade dispatch operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._facade = AcpLocalFacade()
|
||||
|
||||
def time_session_create(self) -> None:
|
||||
"""Benchmark session.create dispatch."""
|
||||
self._facade.dispatch(AcpRequest(operation="session.create", params={}))
|
||||
|
||||
def time_plan_create(self) -> None:
|
||||
"""Benchmark plan.create dispatch."""
|
||||
self._facade.dispatch(AcpRequest(operation="plan.create", params={}))
|
||||
|
||||
def time_plan_execute(self) -> None:
|
||||
"""Benchmark plan.execute dispatch."""
|
||||
self._facade.dispatch(
|
||||
AcpRequest(
|
||||
operation="plan.execute",
|
||||
params={"plan_id": "01M6SM0KE00000000000000001"},
|
||||
)
|
||||
)
|
||||
|
||||
def time_list_operations(self) -> None:
|
||||
"""Benchmark list_operations call."""
|
||||
self._facade.list_operations()
|
||||
|
||||
|
||||
class M6GuardEvaluationSuite:
|
||||
"""Benchmark automation guard evaluation."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._profile_denylist = AutomationProfile(
|
||||
name="bench-deny",
|
||||
guards=AutomationGuard(
|
||||
tool_denylist=["rm_rf", "drop_database", "format_disk"],
|
||||
),
|
||||
)
|
||||
self._profile_allowlist = AutomationProfile(
|
||||
name="bench-allow",
|
||||
guards=AutomationGuard(
|
||||
tool_allowlist=["read_file", "search", "list_dir"],
|
||||
),
|
||||
)
|
||||
self._profile_budget = AutomationProfile(
|
||||
name="bench-budget",
|
||||
guards=AutomationGuard(
|
||||
max_total_cost=100.0,
|
||||
max_tool_calls_per_step=10,
|
||||
),
|
||||
)
|
||||
self._profile_no_guards = AutomationProfile(name="bench-noguard")
|
||||
|
||||
def time_denylist_check_allowed(self) -> None:
|
||||
"""Benchmark denylist check for allowed tool."""
|
||||
self._profile_denylist.check_guard(tool_name="read_file")
|
||||
|
||||
def time_denylist_check_denied(self) -> None:
|
||||
"""Benchmark denylist check for denied tool."""
|
||||
self._profile_denylist.check_guard(tool_name="rm_rf")
|
||||
|
||||
def time_allowlist_check_allowed(self) -> None:
|
||||
"""Benchmark allowlist check for allowed tool."""
|
||||
self._profile_allowlist.check_guard(tool_name="read_file")
|
||||
|
||||
def time_allowlist_check_denied(self) -> None:
|
||||
"""Benchmark allowlist check for unlisted tool."""
|
||||
self._profile_allowlist.check_guard(tool_name="write_file")
|
||||
|
||||
def time_budget_check_under(self) -> None:
|
||||
"""Benchmark budget check under limit."""
|
||||
self._profile_budget.check_guard(
|
||||
tool_name="llm_call",
|
||||
cost_so_far=10.0,
|
||||
calls_so_far=3,
|
||||
)
|
||||
|
||||
def time_no_guards_check(self) -> None:
|
||||
"""Benchmark check with no guards configured."""
|
||||
self._profile_no_guards.check_guard(tool_name="anything")
|
||||
|
||||
|
||||
class M6ProfileResolutionSuite:
|
||||
"""Benchmark AutomationProfileService resolution."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._service = AutomationProfileService()
|
||||
|
||||
def time_resolve_plan_level(self) -> None:
|
||||
"""Benchmark resolution with plan-level override."""
|
||||
self._service.resolve_profile(
|
||||
plan_profile="ci",
|
||||
action_profile="auto",
|
||||
project_profile="manual",
|
||||
)
|
||||
|
||||
def time_resolve_action_level(self) -> None:
|
||||
"""Benchmark resolution with action-level override."""
|
||||
self._service.resolve_profile(
|
||||
plan_profile=None,
|
||||
action_profile="auto",
|
||||
project_profile="manual",
|
||||
)
|
||||
|
||||
def time_resolve_global_default(self) -> None:
|
||||
"""Benchmark resolution falling back to global default."""
|
||||
self._service.resolve_profile(
|
||||
plan_profile=None,
|
||||
action_profile=None,
|
||||
project_profile=None,
|
||||
)
|
||||
|
||||
def time_get_builtin_profile(self) -> None:
|
||||
"""Benchmark looking up a built-in profile."""
|
||||
self._service.get_profile("manual")
|
||||
|
||||
def time_list_profiles(self) -> None:
|
||||
"""Benchmark listing all profiles."""
|
||||
self._service.list_profiles()
|
||||
|
||||
def time_evaluate_guard(self) -> None:
|
||||
"""Benchmark guard evaluation through service."""
|
||||
self._service.evaluate_guard(
|
||||
profile_name="manual",
|
||||
tool_name="read_file",
|
||||
)
|
||||
|
||||
def time_version_negotiation(self) -> None:
|
||||
"""Benchmark ACP version negotiation."""
|
||||
negotiator = AcpVersionNegotiator()
|
||||
negotiator.negotiate("1.0")
|
||||
|
||||
|
||||
class M6EventQueueSuite:
|
||||
"""Benchmark ACP event queue operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._queue = AcpEventQueue()
|
||||
|
||||
def teardown(self) -> None:
|
||||
if not self._queue.is_closed:
|
||||
self._queue.close()
|
||||
|
||||
def time_publish_event(self) -> None:
|
||||
"""Benchmark publishing a single event."""
|
||||
self._queue.publish(AcpEvent(event_type="plan.progress", data={"step": 1}))
|
||||
|
||||
def time_subscribe_and_publish(self) -> None:
|
||||
"""Benchmark subscribe + publish cycle."""
|
||||
q = AcpEventQueue()
|
||||
sub_id = q.subscribe_local(lambda _e: None)
|
||||
q.publish(AcpEvent(event_type="test.event", data={}))
|
||||
q.unsubscribe(sub_id)
|
||||
q.close()
|
||||
|
||||
def time_get_events(self) -> None:
|
||||
"""Benchmark retrieving events from queue."""
|
||||
self._queue.get_events(limit=50)
|
||||
|
||||
|
||||
class M6FixtureLoadSuite:
|
||||
"""Benchmark loading M6 fixture files."""
|
||||
|
||||
def time_load_acp_facade_flows(self) -> None:
|
||||
"""Benchmark loading acp_facade_flows.json."""
|
||||
with open(_FIXTURES_DIR / "acp_facade_flows.json") as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_autonomy_guardrails(self) -> None:
|
||||
"""Benchmark loading autonomy_guardrails.json."""
|
||||
with open(_FIXTURES_DIR / "autonomy_guardrails.json") as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_automation_profiles(self) -> None:
|
||||
"""Benchmark loading automation_profiles.json."""
|
||||
with open(_FIXTURES_DIR / "automation_profiles.json") as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_all_fixtures(self) -> None:
|
||||
"""Benchmark loading all M6 fixture files."""
|
||||
for fname in (
|
||||
"acp_facade_flows.json",
|
||||
"autonomy_guardrails.json",
|
||||
"automation_profiles.json",
|
||||
):
|
||||
with open(_FIXTURES_DIR / fname) as f:
|
||||
json.load(f)
|
||||
+363
-242
@@ -81,12 +81,12 @@ Both messages are single-line and CI-parseable. The CI pipeline greps for these
|
||||
|
||||
Three report formats are generated under `build/`:
|
||||
|
||||
| Report | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| Terminal | stdout | Per-file summary with `--show-missing` |
|
||||
| HTML | `build/htmlcov/index.html` | Interactive browser report |
|
||||
| XML | `build/coverage.xml` | Cobertura-format for CI tools |
|
||||
| JSON | `build/coverage.json` | Machine-readable totals and per-file data |
|
||||
| Report | Path | Description |
|
||||
| -------- | -------------------------- | ----------------------------------------- |
|
||||
| Terminal | stdout | Per-file summary with `--show-missing` |
|
||||
| HTML | `build/htmlcov/index.html` | Interactive browser report |
|
||||
| XML | `build/coverage.xml` | Cobertura-format for CI tools |
|
||||
| JSON | `build/coverage.json` | Machine-readable totals and per-file data |
|
||||
|
||||
### Coverage Configuration
|
||||
|
||||
@@ -258,16 +258,16 @@ outcomes.
|
||||
|
||||
Covers 68 scenarios across these areas:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Action CRUD | 13 | create, list, show, archive with success + error paths |
|
||||
| Plan Use | 10 | basic use, multi-project, automation profile, invariants, actor overrides |
|
||||
| Plan Status | 9 | phase visibility (strategize, execute, apply) and terminal outcomes (applied, constrained, errored, cancelled) |
|
||||
| Plan Execute | 5 | execute success, phase transition, error paths |
|
||||
| Plan Apply | 5 | lifecycle-apply success, terminal outcomes, error paths |
|
||||
| Plan Cancel | 5 | cancel with reason, already-terminal guard, error paths |
|
||||
| Plan List | 4 | lifecycle-list with and without filters |
|
||||
| Negative cases | 17 | missing config, invalid args, invalid project names, unknown actions/resources |
|
||||
| Area | Scenarios | Description |
|
||||
| -------------- | --------- | -------------------------------------------------------------------------------------------------------------- |
|
||||
| Action CRUD | 13 | create, list, show, archive with success + error paths |
|
||||
| Plan Use | 10 | basic use, multi-project, automation profile, invariants, actor overrides |
|
||||
| Plan Status | 9 | phase visibility (strategize, execute, apply) and terminal outcomes (applied, constrained, errored, cancelled) |
|
||||
| Plan Execute | 5 | execute success, phase transition, error paths |
|
||||
| Plan Apply | 5 | lifecycle-apply success, terminal outcomes, error paths |
|
||||
| Plan Cancel | 5 | cancel with reason, already-terminal guard, error paths |
|
||||
| Plan List | 4 | lifecycle-list with and without filters |
|
||||
| Negative cases | 17 | missing config, invalid args, invalid project names, unknown actions/resources |
|
||||
|
||||
Step definitions: `features/steps/cli_lifecycle_coverage_steps.py`
|
||||
|
||||
@@ -281,16 +281,16 @@ the CLI entry points. Uses `robot/helper_cli_lifecycle_e2e.py` as a Python helpe
|
||||
that mocks the service layer while exercising real CLI argument parsing and output
|
||||
formatting.
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Action Create From Config Via CLI | Creates an action from YAML config |
|
||||
| Plan Use Creates Plan In Strategize Phase | Uses an action to create a plan |
|
||||
| Plan Execute Transitions To Execute Phase | Executes a plan, checks phase |
|
||||
| Plan Lifecycle Apply Transitions To Apply Phase | Applies a plan, checks phase |
|
||||
| Plan Status Shows Plan Details | Verifies status output rendering |
|
||||
| Plan Cancel Cancels Non-Terminal Plan | Cancels with a reason string |
|
||||
| Full Lifecycle Action To Apply | End-to-end: create -> use -> execute -> apply |
|
||||
| Plan Lifecycle List Shows Plans | Verifies lifecycle-list output |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------------------- | --------------------------------------------- |
|
||||
| Action Create From Config Via CLI | Creates an action from YAML config |
|
||||
| Plan Use Creates Plan In Strategize Phase | Uses an action to create a plan |
|
||||
| Plan Execute Transitions To Execute Phase | Executes a plan, checks phase |
|
||||
| Plan Lifecycle Apply Transitions To Apply Phase | Applies a plan, checks phase |
|
||||
| Plan Status Shows Plan Details | Verifies status output rendering |
|
||||
| Plan Cancel Cancels Non-Terminal Plan | Cancels with a reason string |
|
||||
| Full Lifecycle Action To Apply | End-to-end: create -> use -> execute -> apply |
|
||||
| Plan Lifecycle List Shows Plans | Verifies lifecycle-list output |
|
||||
|
||||
### ASV Benchmark: `benchmarks/plan_cli_smoke_bench.py`
|
||||
|
||||
@@ -328,21 +328,21 @@ for change tracking.
|
||||
|
||||
**Positive tests:**
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Action Create Via Config CLI | Creates an action from YAML config |
|
||||
| Plan Use Creates Strategize Plan With Sandbox | Uses action with sandbox workspace mock |
|
||||
| Plan Execute With ChangeSet Capture | Executes plan, verifies changeset entry recorded |
|
||||
| Plan Apply With ChangeSet Verification | Applies plan, verifies changeset summary |
|
||||
| Full Lifecycle Action To Apply With Sandbox | End-to-end with sandbox + changeset |
|
||||
| Test Case | Description |
|
||||
| --------------------------------------------- | ------------------------------------------------ |
|
||||
| Action Create Via Config CLI | Creates an action from YAML config |
|
||||
| Plan Use Creates Strategize Plan With Sandbox | Uses action with sandbox workspace mock |
|
||||
| Plan Execute With ChangeSet Capture | Executes plan, verifies changeset entry recorded |
|
||||
| Plan Apply With ChangeSet Verification | Applies plan, verifies changeset summary |
|
||||
| Full Lifecycle Action To Apply With Sandbox | End-to-end with sandbox + changeset |
|
||||
|
||||
**Negative tests:**
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Invalid Project Name Rejected | Invalid project name format handled gracefully |
|
||||
| Missing Resource Returns Error | Non-existent action returns error |
|
||||
| Invalid Arg Format Rejected | `--arg` without `=` separator rejected |
|
||||
| Test Case | Description |
|
||||
| ------------------------------ | ---------------------------------------------- |
|
||||
| Invalid Project Name Rejected | Invalid project name format handled gracefully |
|
||||
| Missing Resource Returns Error | Non-existent action returns error |
|
||||
| Invalid Arg Format Rejected | `--arg` without `=` separator rejected |
|
||||
|
||||
**Fixtures:**
|
||||
|
||||
@@ -403,13 +403,13 @@ Step definitions: `features/steps/action_persistence_steps.py`
|
||||
|
||||
5 end-to-end tests exercising the persistence layer through Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Plan Full Lifecycle Persistence | Create action + plan, transition through all phases to `applied` terminal state |
|
||||
| Plan Restart Persistence | Create plan, close DB, reopen, verify all fields survive |
|
||||
| Plan Concurrent Session Access | Two independent sessions against the same DB file |
|
||||
| Action CRUD Persistence E2E | Create, read, update, delete an action with arguments/invariants |
|
||||
| Plan Tree Hierarchy Persistence E2E | Parent/child plan hierarchy with `root_plan_id` verification |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------- | ------------------------------------------------------------------------------- |
|
||||
| Plan Full Lifecycle Persistence | Create action + plan, transition through all phases to `applied` terminal state |
|
||||
| Plan Restart Persistence | Create plan, close DB, reopen, verify all fields survive |
|
||||
| Plan Concurrent Session Access | Two independent sessions against the same DB file |
|
||||
| Action CRUD Persistence E2E | Create, read, update, delete an action with arguments/invariants |
|
||||
| Plan Tree Hierarchy Persistence E2E | Parent/child plan hierarchy with `root_plan_id` verification |
|
||||
|
||||
Helper script: `robot/helper_plan_persistence_e2e.py`
|
||||
|
||||
@@ -417,15 +417,15 @@ Helper script: `robot/helper_plan_persistence_e2e.py`
|
||||
|
||||
7 end-to-end tests exercising persistence lifecycle patterns through a Python helper:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Plan Full Lifecycle Phase Transitions | Create action + plan, transition through all phases to `applied` terminal state |
|
||||
| Process Restart Simulation | Write plan, close DB, reopen from disk, verify all fields survive |
|
||||
| Reopen Plan Status After Restart | Close/reopen, verify phase, state, automation level, and tags unchanged |
|
||||
| Concurrent CLI Access Safeguards | Two independent sessions see each other's committed data |
|
||||
| Stored Arguments Ordering Persistence | 4 ordered `ActionArgument` entries survive roundtrip |
|
||||
| Stored Invariants Ordering Persistence | 3 ordered invariant strings survive roundtrip |
|
||||
| Project Links Persistence Through Restart | 2 `ProjectLink` entries survive close/reopen cycle |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------------- | ------------------------------------------------------------------------------- |
|
||||
| Plan Full Lifecycle Phase Transitions | Create action + plan, transition through all phases to `applied` terminal state |
|
||||
| Process Restart Simulation | Write plan, close DB, reopen from disk, verify all fields survive |
|
||||
| Reopen Plan Status After Restart | Close/reopen, verify phase, state, automation level, and tags unchanged |
|
||||
| Concurrent CLI Access Safeguards | Two independent sessions see each other's committed data |
|
||||
| Stored Arguments Ordering Persistence | 4 ordered `ActionArgument` entries survive roundtrip |
|
||||
| Stored Invariants Ordering Persistence | 3 ordered invariant strings survive roundtrip |
|
||||
| Project Links Persistence Through Restart | 2 `ProjectLink` entries survive close/reopen cycle |
|
||||
|
||||
Helper script: `robot/helper_persistence_lifecycle.py`
|
||||
|
||||
@@ -537,13 +537,13 @@ times, rolling back previously flushed but uncommitted data.
|
||||
|
||||
5 smoke tests exercising the registry through a Python helper script:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Register And Retrieve A Skill | Round-trip: register then get by name |
|
||||
| Test Case | Description |
|
||||
| --------------------------------- | --------------------------------------------- |
|
||||
| Register And Retrieve A Skill | Round-trip: register then get by name |
|
||||
| List Skills With Namespace Filter | Register multiple skills, filter by namespace |
|
||||
| Update A Skill | Change description after registration |
|
||||
| Reject Duplicate Skill Name | Duplicate name produces error |
|
||||
| Remove A Skill | Remove and verify absence |
|
||||
| Update A Skill | Change description after registration |
|
||||
| Reject Duplicate Skill Name | Duplicate name produces error |
|
||||
| Remove A Skill | Remove and verify absence |
|
||||
|
||||
Helper script: `robot/helper_skill_registry.py`
|
||||
|
||||
@@ -601,17 +601,17 @@ suites at unit, integration, and benchmark levels.
|
||||
33 scenarios covering Plan hierarchy properties, SubplanConfig defaults, SubplanStatus
|
||||
tracking, SubplanFailureHandler decisions, and CLI dict rendering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Root plan identity | 2 | `is_root_plan`, `is_subplan`, `depth` for root plans |
|
||||
| Child plan identity | 3 | Parent reference, attempt counter, root propagation |
|
||||
| Default values | 4 | Phase, state, automation level, subplan_config defaults |
|
||||
| SubplanConfig storage | 3 | Sequential/parallel config, standalone defaults |
|
||||
| Hierarchy helpers | 3 | `has_subplans`, `depth` for non-root plans |
|
||||
| Lifecycle transitions | 2 | Child plans follow same phase transition rules |
|
||||
| Dependency guardrails | 5 | FailureHandler stop/retry decisions, phase independence |
|
||||
| CLI dict rendering | 3 | `parent_plan_id`, `subplan_count` in output |
|
||||
| Validation guardrails | 4 | Invalid ULIDs, out-of-range config values |
|
||||
| Area | Scenarios | Description |
|
||||
| --------------------- | --------- | ------------------------------------------------------- |
|
||||
| Root plan identity | 2 | `is_root_plan`, `is_subplan`, `depth` for root plans |
|
||||
| Child plan identity | 3 | Parent reference, attempt counter, root propagation |
|
||||
| Default values | 4 | Phase, state, automation level, subplan_config defaults |
|
||||
| SubplanConfig storage | 3 | Sequential/parallel config, standalone defaults |
|
||||
| Hierarchy helpers | 3 | `has_subplans`, `depth` for non-root plans |
|
||||
| Lifecycle transitions | 2 | Child plans follow same phase transition rules |
|
||||
| Dependency guardrails | 5 | FailureHandler stop/retry decisions, phase independence |
|
||||
| CLI dict rendering | 3 | `parent_plan_id`, `subplan_count` in output |
|
||||
| Validation guardrails | 4 | Invalid ULIDs, out-of-range config values |
|
||||
|
||||
Step definitions: `features/steps/subplan_model_steps.py`
|
||||
|
||||
@@ -622,19 +622,19 @@ with existing `plan_model_steps.py` steps.
|
||||
|
||||
11 smoke tests exercising subplan model properties via Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Root Plan Identity Flags | `is_root_plan=True`, `is_subplan=False`, `depth=0` |
|
||||
| Child Plan Identity Flags | `is_subplan=True`, `is_root_plan=False`, `depth=-1` |
|
||||
| Three Level Hierarchy Root Propagation | `root_plan_id` propagates through 3 levels |
|
||||
| SubplanConfig Defaults | All default values are sensible |
|
||||
| SubplanConfig Custom Settings | Parallel mode with custom retry settings |
|
||||
| Failure Handler Stop Others On Fail Fast | `should_stop_others` with `fail_fast=True` |
|
||||
| Failure Handler Retry Retriable Error | `should_retry` for `TimeoutError` |
|
||||
| Failure Handler Skip Non Retriable Error | `should_retry` skips `ConfigurationError` |
|
||||
| CLI Dict Renders Parent Plan Id For Child | `parent_plan_id` in child plan CLI dict |
|
||||
| CLI Dict Renders Subplan Count | `subplan_count` in parent plan CLI dict |
|
||||
| Plan With Subplan Config And Statuses | Full plan model with config + statuses |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------------- | --------------------------------------------------- |
|
||||
| Root Plan Identity Flags | `is_root_plan=True`, `is_subplan=False`, `depth=0` |
|
||||
| Child Plan Identity Flags | `is_subplan=True`, `is_root_plan=False`, `depth=-1` |
|
||||
| Three Level Hierarchy Root Propagation | `root_plan_id` propagates through 3 levels |
|
||||
| SubplanConfig Defaults | All default values are sensible |
|
||||
| SubplanConfig Custom Settings | Parallel mode with custom retry settings |
|
||||
| Failure Handler Stop Others On Fail Fast | `should_stop_others` with `fail_fast=True` |
|
||||
| Failure Handler Retry Retriable Error | `should_retry` for `TimeoutError` |
|
||||
| Failure Handler Skip Non Retriable Error | `should_retry` skips `ConfigurationError` |
|
||||
| CLI Dict Renders Parent Plan Id For Child | `parent_plan_id` in child plan CLI dict |
|
||||
| CLI Dict Renders Subplan Count | `subplan_count` in parent plan CLI dict |
|
||||
| Plan With Subplan Config And Statuses | Full plan model with config + statuses |
|
||||
|
||||
Helper script: `robot/helper_subplan_model.py`
|
||||
|
||||
@@ -689,6 +689,7 @@ All jobs use Python 3.13 and route commands through `nox`. See `docs/development
|
||||
Ensure you're running in serial mode (not parallel). The `coverage_report` nox session handles this correctly. Do not run `behave` directly outside of `coverage run`.
|
||||
|
||||
Also ensure:
|
||||
|
||||
- `parallel = false` is set in `[tool.coverage.run]`
|
||||
- `COVERAGE_FILE` and `COVERAGE_RCFILE` env vars are set correctly by the nox session
|
||||
|
||||
@@ -740,13 +741,13 @@ conflict detection across the validation subsystem.
|
||||
|
||||
### Shared Fixtures (`features/fixtures/validation/`)
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `malformed_tool_output.json` | Missing required fields, wrong types, extra nulls, non-object outputs |
|
||||
| `missing_resources.json` | Dangling references, unresolvable paths, circular dependencies |
|
||||
| `validation_timeouts.json` | Slow validators, zero/negative timeouts, partial results |
|
||||
| `invalid_schema_transforms.json` | Transforms returning non-dict, missing type, null, circular refs |
|
||||
| `mixed_ordering.json` | Required vs informational ordering, duplicate attachment IDs |
|
||||
| Fixture File | Description |
|
||||
| -------------------------------- | --------------------------------------------------------------------- |
|
||||
| `malformed_tool_output.json` | Missing required fields, wrong types, extra nulls, non-object outputs |
|
||||
| `missing_resources.json` | Dangling references, unresolvable paths, circular dependencies |
|
||||
| `validation_timeouts.json` | Slow validators, zero/negative timeouts, partial results |
|
||||
| `invalid_schema_transforms.json` | Transforms returning non-dict, missing type, null, circular refs |
|
||||
| `mixed_ordering.json` | Required vs informational ordering, duplicate attachment IDs |
|
||||
|
||||
Each fixture file is a JSON document with a `"fixtures"` array. Each entry has
|
||||
`"name"`, `"description"`, `"input"`, and `"expected_error"` fields.
|
||||
@@ -755,15 +756,15 @@ Each fixture file is a JSON document with a `"fixtures"` array. Each entry has
|
||||
|
||||
25 scenarios covering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|---|---|---|
|
||||
| Malformed tool output | 6 | Missing fields, wrong types, nulls, empty/string outputs |
|
||||
| Missing resources | 5 | Dangling actions, unresolvable paths, missing actors, circular deps |
|
||||
| Validation timeouts | 4 | Slow validators, zero timeout, negative timeout, partial results |
|
||||
| Invalid schema transforms | 3 | Non-dict returns, missing type field, null schema |
|
||||
| Mixed ordering | 4 | Required-before-informational, duplicate attachment IDs, unknown levels |
|
||||
| Concurrent conflicts | 1 | Two validators on same resource detect conflict |
|
||||
| Rollback on failure | 2 | Required failure triggers rollback of pending changes |
|
||||
| Area | Scenarios | Description |
|
||||
| ------------------------- | --------- | ----------------------------------------------------------------------- |
|
||||
| Malformed tool output | 6 | Missing fields, wrong types, nulls, empty/string outputs |
|
||||
| Missing resources | 5 | Dangling actions, unresolvable paths, missing actors, circular deps |
|
||||
| Validation timeouts | 4 | Slow validators, zero timeout, negative timeout, partial results |
|
||||
| Invalid schema transforms | 3 | Non-dict returns, missing type field, null schema |
|
||||
| Mixed ordering | 4 | Required-before-informational, duplicate attachment IDs, unknown levels |
|
||||
| Concurrent conflicts | 1 | Two validators on same resource detect conflict |
|
||||
| Rollback on failure | 2 | Required failure triggers rollback of pending changes |
|
||||
|
||||
Step definitions: `features/steps/validation_edge_case_steps.py`
|
||||
|
||||
@@ -774,14 +775,14 @@ conflicts with existing steps.
|
||||
|
||||
6 integration tests exercising the validation edge case helper:
|
||||
|
||||
| Test Case | Description |
|
||||
|---|---|
|
||||
| Validation Edge Load All Fixtures | Loads all fixture files and verifies structure |
|
||||
| Validation Edge Malformed Output Detection | Detects malformed tool output patterns |
|
||||
| Validation Edge Missing Resource Error Paths | Validates missing resource reference detection |
|
||||
| Validation Edge Timeout Simulation | Validates timeout simulation fixture data |
|
||||
| Validation Edge Schema Validation Errors | Validates schema transform error detection |
|
||||
| Validation Edge Unknown Command Returns Error | Verifies unknown command exits with code 1 |
|
||||
| Test Case | Description |
|
||||
| --------------------------------------------- | ---------------------------------------------- |
|
||||
| Validation Edge Load All Fixtures | Loads all fixture files and verifies structure |
|
||||
| Validation Edge Malformed Output Detection | Detects malformed tool output patterns |
|
||||
| Validation Edge Missing Resource Error Paths | Validates missing resource reference detection |
|
||||
| Validation Edge Timeout Simulation | Validates timeout simulation fixture data |
|
||||
| Validation Edge Schema Validation Errors | Validates schema transform error detection |
|
||||
| Validation Edge Unknown Command Returns Error | Verifies unknown command exits with code 1 |
|
||||
|
||||
Helper script: `robot/helper_validation_edge.py`
|
||||
|
||||
@@ -838,11 +839,11 @@ nox -s benchmark
|
||||
|
||||
### Scale Test Suites
|
||||
|
||||
| Suite | Framework | Scenarios | Description |
|
||||
|-------|-----------|-----------|-------------|
|
||||
| `features/scale_test.feature` | Behave | 20 | Fixture loading, metadata validation, threshold checks, distribution simulation |
|
||||
| `robot/scale_test.robot` | Robot | 6 | Integration-level fixture validation via helper script |
|
||||
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
|
||||
| Suite | Framework | Scenarios | Description |
|
||||
| ----------------------------------- | --------- | --------- | ------------------------------------------------------------------------------- |
|
||||
| `features/scale_test.feature` | Behave | 20 | Fixture loading, metadata validation, threshold checks, distribution simulation |
|
||||
| `robot/scale_test.robot` | Robot | 6 | Integration-level fixture validation via helper script |
|
||||
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
|
||||
|
||||
For full scale testing documentation, see `docs/development/scale_testing.md`.
|
||||
|
||||
@@ -861,12 +862,12 @@ rendering without requiring a database or running server.
|
||||
|
||||
#### Common Fixtures
|
||||
|
||||
| Fixture | Module | Description |
|
||||
|---------|--------|-------------|
|
||||
| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides |
|
||||
| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) |
|
||||
| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead |
|
||||
| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations |
|
||||
| Fixture | Module | Description |
|
||||
| ------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides |
|
||||
| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) |
|
||||
| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead |
|
||||
| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations |
|
||||
|
||||
#### Automation Profile Resolution Fixtures
|
||||
|
||||
@@ -886,6 +887,7 @@ repeated option handling.
|
||||
|
||||
Tests exercise the `validate_namespaced_actor` regex
|
||||
(`^[a-z][a-z0-9-]*/[a-z][a-z0-9._-]*$`) with various invalid inputs:
|
||||
|
||||
- Empty string
|
||||
- Missing slash (`no-slash`)
|
||||
- Uppercase characters (`UPPER/case`)
|
||||
@@ -900,6 +902,7 @@ Tests exercise the `validate_namespaced_actor` regex
|
||||
|
||||
Tests render plan status, lifecycle-list, and action show output in JSON, YAML,
|
||||
and table formats, then assert that:
|
||||
|
||||
- JSON output is valid (parses without error)
|
||||
- Required keys are present (`namespaced_name`, `phase`, `automation_profile`, etc.)
|
||||
- Invariant text values survive serialization round-trips
|
||||
@@ -910,21 +913,21 @@ and table formats, then assert that:
|
||||
|
||||
Covers scenarios across these areas:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Automation profile flags | 2 | Valid and invalid profile names |
|
||||
| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty |
|
||||
| Invariant flags | 2 | Single and dual invariant flags |
|
||||
| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation |
|
||||
| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors |
|
||||
| Actor overrides (invalid) | 4 | Malformed namespace/name formats |
|
||||
| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace |
|
||||
| Plan status display | 4 | Table, JSON, single plan with invariants/profile |
|
||||
| Plan lifecycle-list | 2 | Table and JSON with invariant count |
|
||||
| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile |
|
||||
| Combined flags | 1 | Profile + invariant together |
|
||||
| validate_namespaced_actor | 2 | Unit-level accept/reject assertions |
|
||||
| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action |
|
||||
| Area | Scenarios | Description |
|
||||
| ------------------------- | --------- | -------------------------------------------------------------------- |
|
||||
| Automation profile flags | 2 | Valid and invalid profile names |
|
||||
| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty |
|
||||
| Invariant flags | 2 | Single and dual invariant flags |
|
||||
| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation |
|
||||
| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors |
|
||||
| Actor overrides (invalid) | 4 | Malformed namespace/name formats |
|
||||
| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace |
|
||||
| Plan status display | 4 | Table, JSON, single plan with invariants/profile |
|
||||
| Plan lifecycle-list | 2 | Table and JSON with invariant count |
|
||||
| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile |
|
||||
| Combined flags | 1 | Profile + invariant together |
|
||||
| validate_namespaced_actor | 2 | Unit-level accept/reject assertions |
|
||||
| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action |
|
||||
|
||||
Step definitions: `features/steps/cli_extensions_steps.py`
|
||||
|
||||
@@ -932,17 +935,17 @@ Step definitions: `features/steps/cli_extensions_steps.py`
|
||||
|
||||
Integration tests exercising CLI extensions through Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly |
|
||||
| Plan Use With Automation Profile | Verifies `--automation-profile` works |
|
||||
| Plan Use Actor Validation Valid | Valid namespaced actor format accepted |
|
||||
| Plan Use Actor Validation Invalid | Invalid actor format rejected |
|
||||
| Plan Use Combined Profile And Invariants | Both profile and invariants together |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly |
|
||||
| Plan Use With Automation Profile | Verifies `--automation-profile` works |
|
||||
| Plan Use Actor Validation Valid | Valid namespaced actor format accepted |
|
||||
| Plan Use Actor Validation Invalid | Invalid actor format rejected |
|
||||
| Plan Use Combined Profile And Invariants | Both profile and invariants together |
|
||||
| Action Show With Optional Actors And Invariants | `action show` renders estimation_actor, invariant_actor, invariants, inputs_schema |
|
||||
| Action Show JSON With Optional Fields | JSON output includes all optional fields |
|
||||
| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order |
|
||||
| Profile Resolution All Builtins | All builtin profile names resolve correctly |
|
||||
| Action Show JSON With Optional Fields | JSON output includes all optional fields |
|
||||
| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order |
|
||||
| Profile Resolution All Builtins | All builtin profile names resolve correctly |
|
||||
|
||||
Helper script: `robot/helper_cli_extensions.py`
|
||||
|
||||
@@ -951,6 +954,7 @@ Helper script: `robot/helper_cli_extensions.py`
|
||||
Two benchmark files cover CLI extension performance:
|
||||
|
||||
**`benchmarks/cli_extensions_bench.py`** (original):
|
||||
|
||||
- `PlanUseWithProfileSuite` — plan use with automation profile
|
||||
- `PlanUseWithInvariantsSuite` — plan use with invariant flags
|
||||
- `PlanUseActorValidationSuite` — plan use with actor overrides
|
||||
@@ -958,6 +962,7 @@ Two benchmark files cover CLI extension performance:
|
||||
- `ActionShowExtendedSuite` — action show in rich and JSON formats
|
||||
|
||||
**`benchmarks/cli_extension_tests_bench.py`** (extended #326):
|
||||
|
||||
- `ProfileResolutionSuite` — resolve all builtin profiles, single profile, invalid profile
|
||||
- `InvariantOrderingSuite` — three and five invariant flag parsing
|
||||
- `ActorValidationErrorSuite` — valid/invalid actor regex, CLI rejection
|
||||
@@ -984,10 +989,10 @@ The M2 smoke suite validates the foundation for the Actor Graphs + Tool Sources
|
||||
|
||||
M2 fixtures live in `features/fixtures/m2/`:
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| Fixture File | Description |
|
||||
| ---------------------------- | --------------------------------------------------------- |
|
||||
| `m2_hierarchical_actor.yaml` | Graph-type actor with planner → executor → reviewer nodes |
|
||||
| `m2_skill_pack.yaml` | Skill pack with tool references and an inline tool |
|
||||
| `m2_skill_pack.yaml` | Skill pack with tool references and an inline tool |
|
||||
|
||||
### MCP Stub Server
|
||||
|
||||
@@ -1007,11 +1012,11 @@ server.stop()
|
||||
|
||||
### Test Suites
|
||||
|
||||
| Suite | Framework | Scenarios | Description |
|
||||
|-------|-----------|-----------|-------------|
|
||||
| `features/m2_actor_tool_smoke.feature` | Behave | 10 | Actor loading, skill registry, tool lifecycle, MCP stub |
|
||||
| `robot/m2_actor_tool_smoke.robot` | Robot | 6 | CLI-level smoke for actor/skill/tool/MCP operations |
|
||||
| `benchmarks/m2_actor_tool_smoke_bench.py` | ASV | 12 | Baseline runtime for actor, skill, tool, and MCP operations |
|
||||
| Suite | Framework | Scenarios | Description |
|
||||
| ----------------------------------------- | --------- | --------- | ----------------------------------------------------------- |
|
||||
| `features/m2_actor_tool_smoke.feature` | Behave | 10 | Actor loading, skill registry, tool lifecycle, MCP stub |
|
||||
| `robot/m2_actor_tool_smoke.robot` | Robot | 6 | CLI-level smoke for actor/skill/tool/MCP operations |
|
||||
| `benchmarks/m2_actor_tool_smoke_bench.py` | ASV | 12 | Baseline runtime for actor, skill, tool, and MCP operations |
|
||||
|
||||
### Running the M2 Smoke Suite
|
||||
|
||||
@@ -1043,26 +1048,26 @@ link resources, and capture plan IDs for subsequent CLI verification.
|
||||
|
||||
### Fixtures (`features/fixtures/m1/`)
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `git_repo.json` | Minimal git repo definitions with file listings and branch info |
|
||||
| `git_checkout_resource.json` | Git-checkout resource configs with path and branch properties |
|
||||
| `action_sourcecode.yaml` | Minimal action YAML with strategy/execution actors and arguments |
|
||||
| Fixture File | Description |
|
||||
| ---------------------------- | ---------------------------------------------------------------- |
|
||||
| `git_repo.json` | Minimal git repo definitions with file listings and branch info |
|
||||
| `git_checkout_resource.json` | Git-checkout resource configs with path and branch properties |
|
||||
| `action_sourcecode.yaml` | Minimal action YAML with strategy/execution actors and arguments |
|
||||
|
||||
### Behave Suite: `features/m1_sourcecode_smoke.feature`
|
||||
|
||||
16 scenarios covering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Fixture loading | 3 | Load and validate git repo, checkout resource, action YAML |
|
||||
| Action create | 1 | Create action from YAML config via CLI |
|
||||
| Project/resource | 2 | Create temp project, link resource |
|
||||
| Plan use | 3 | Use action to create plan, with project and args |
|
||||
| Plan execute | 1 | Execute transitions to execute phase |
|
||||
| Plan diff | 1 | Show changeset via diff |
|
||||
| Plan apply | 1 | Apply transitions to terminal state |
|
||||
| Negative cases | 3 | Unknown action, wrong phase, invalid actor |
|
||||
| Area | Scenarios | Description |
|
||||
| ---------------- | --------- | ---------------------------------------------------------- |
|
||||
| Fixture loading | 3 | Load and validate git repo, checkout resource, action YAML |
|
||||
| Action create | 1 | Create action from YAML config via CLI |
|
||||
| Project/resource | 2 | Create temp project, link resource |
|
||||
| Plan use | 3 | Use action to create plan, with project and args |
|
||||
| Plan execute | 1 | Execute transitions to execute phase |
|
||||
| Plan diff | 1 | Show changeset via diff |
|
||||
| Plan apply | 1 | Apply transitions to terminal state |
|
||||
| Negative cases | 3 | Unknown action, wrong phase, invalid actor |
|
||||
|
||||
Step definitions: `features/steps/m1_sourcecode_smoke_steps.py`
|
||||
|
||||
@@ -1073,16 +1078,16 @@ conflicts with existing steps.
|
||||
|
||||
8 integration tests exercising the M1 lifecycle through CLI:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| M1 Action Create From Config | Creates action from fixture YAML |
|
||||
| M1 Plan Use Creates Strategize Plan | Uses action to create plan |
|
||||
| M1 Plan Use With Project Link | Plan use with project argument |
|
||||
| M1 Plan Execute Transitions Phase | Execute phase transition |
|
||||
| M1 Plan Diff Shows Changeset | Diff command for changeset |
|
||||
| M1 Plan Apply Reaches Terminal | Apply to terminal state |
|
||||
| M1 Full Lifecycle Action To Apply | End-to-end flow |
|
||||
| M1 Plan Use With Plain Format | `--format plain` for stable assertions |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------- | -------------------------------------- |
|
||||
| M1 Action Create From Config | Creates action from fixture YAML |
|
||||
| M1 Plan Use Creates Strategize Plan | Uses action to create plan |
|
||||
| M1 Plan Use With Project Link | Plan use with project argument |
|
||||
| M1 Plan Execute Transitions Phase | Execute phase transition |
|
||||
| M1 Plan Diff Shows Changeset | Diff command for changeset |
|
||||
| M1 Plan Apply Reaches Terminal | Apply to terminal state |
|
||||
| M1 Full Lifecycle Action To Apply | End-to-end flow |
|
||||
| M1 Plan Use With Plain Format | `--format plain` for stable assertions |
|
||||
|
||||
Helper script: `robot/helper_m1_sourcecode_smoke.py`
|
||||
|
||||
@@ -1116,17 +1121,17 @@ nox -s benchmark
|
||||
To perform a quick M1 source-code smoke run:
|
||||
|
||||
1. Run the Behave feature:
|
||||
```bash
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
```
|
||||
```bash
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
```
|
||||
2. Run the Robot suite:
|
||||
```bash
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
```
|
||||
```bash
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
```
|
||||
3. Run benchmarks:
|
||||
```bash
|
||||
nox -s benchmark
|
||||
```
|
||||
```bash
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### Failure Triage Tips
|
||||
|
||||
@@ -1156,16 +1161,16 @@ Issue: [#179](https://git.cleverthis.com/cleveragents/cleveragents-core/issues/1
|
||||
|
||||
### Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|------|
|
||||
| `features/m3_decision_validation_smoke.feature` | Behave BDD scenarios (25 scenarios) |
|
||||
| `features/steps/m3_decision_validation_smoke_steps.py` | Step implementations |
|
||||
| `features/fixtures/m3/decision_tree_outputs.json` | Decision tree fixture data |
|
||||
| `features/fixtures/m3/validation_attachments.json` | Validation attachment fixture data |
|
||||
| `features/fixtures/m3/invariant_configs.json` | Invariant config fixture data |
|
||||
| `robot/m3_decision_validation_smoke.robot` | Robot Framework integration tests (8 cases) |
|
||||
| `robot/helper_m3_decision_validation_smoke.py` | Robot helper with 8 subcommands |
|
||||
| `benchmarks/m3_smoke_bench.py` | ASV benchmark suites (5 classes) |
|
||||
| File | Purpose |
|
||||
| ------------------------------------------------------ | ------------------------------------------- |
|
||||
| `features/m3_decision_validation_smoke.feature` | Behave BDD scenarios (25 scenarios) |
|
||||
| `features/steps/m3_decision_validation_smoke_steps.py` | Step implementations |
|
||||
| `features/fixtures/m3/decision_tree_outputs.json` | Decision tree fixture data |
|
||||
| `features/fixtures/m3/validation_attachments.json` | Validation attachment fixture data |
|
||||
| `features/fixtures/m3/invariant_configs.json` | Invariant config fixture data |
|
||||
| `robot/m3_decision_validation_smoke.robot` | Robot Framework integration tests (8 cases) |
|
||||
| `robot/helper_m3_decision_validation_smoke.py` | Robot helper with 8 subcommands |
|
||||
| `benchmarks/m3_smoke_bench.py` | ASV benchmark suites (5 classes) |
|
||||
|
||||
### Scenario Categories
|
||||
|
||||
@@ -1213,24 +1218,24 @@ sequential-apply strategies).
|
||||
|
||||
### Fixtures (`features/fixtures/m4/`)
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `correction_flows.json` | Revert, append, dry-run, and high-risk correction scenarios |
|
||||
| `subplan_execution.json` | Sequential, parallel, dependency-ordered, and retry subplan configs |
|
||||
| `conflict_simulations.json` | Merge conflict scenarios for all SubplanMergeStrategy variants |
|
||||
| Fixture File | Description |
|
||||
| --------------------------- | ------------------------------------------------------------------- |
|
||||
| `correction_flows.json` | Revert, append, dry-run, and high-risk correction scenarios |
|
||||
| `subplan_execution.json` | Sequential, parallel, dependency-ordered, and retry subplan configs |
|
||||
| `conflict_simulations.json` | Merge conflict scenarios for all SubplanMergeStrategy variants |
|
||||
|
||||
### Behave Suite: `features/m4_correction_subplan_smoke.feature`
|
||||
|
||||
20 scenarios covering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Fixture loading | 3 | Load and validate correction, subplan, and conflict fixtures |
|
||||
| Correction flows | 4 | Revert, append, dry-run, and invalid mode via CLI |
|
||||
| Subplan execution | 2 | Sequential and parallel subplan status rendering |
|
||||
| Failure handling | 3 | Stop-others, retry retriable, skip non-retriable |
|
||||
| Conflict simulation | 3 | No-conflict, fail-on-conflict, last-wins merge strategies |
|
||||
| Negative cases | 2 | Non-existent plan, empty decision ID |
|
||||
| Area | Scenarios | Description |
|
||||
| ------------------- | --------- | ------------------------------------------------------------ |
|
||||
| Fixture loading | 3 | Load and validate correction, subplan, and conflict fixtures |
|
||||
| Correction flows | 4 | Revert, append, dry-run, and invalid mode via CLI |
|
||||
| Subplan execution | 2 | Sequential and parallel subplan status rendering |
|
||||
| Failure handling | 3 | Stop-others, retry retriable, skip non-retriable |
|
||||
| Conflict simulation | 3 | No-conflict, fail-on-conflict, last-wins merge strategies |
|
||||
| Negative cases | 2 | Non-existent plan, empty decision ID |
|
||||
|
||||
Step definitions: `features/steps/m4_correction_subplan_smoke_steps.py`
|
||||
|
||||
@@ -1241,15 +1246,15 @@ conflicts with existing steps.
|
||||
|
||||
8 integration tests exercising M4 correction and subplan flows through CLI:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| M4 Correction Revert Via CLI | Invoke plan correct --mode revert |
|
||||
| M4 Correction Append Via CLI | Invoke plan correct --mode append |
|
||||
| M4 Correction Dry Run Via CLI | Invoke plan correct --dry-run |
|
||||
| M4 Subplan Status Sequential | Plan status with sequential subplan config |
|
||||
| M4 Subplan Status Parallel | Plan status with parallel subplan config |
|
||||
| M4 Failure Handler Evaluation | SubplanFailureHandler decision logic |
|
||||
| M4 Fixture Loading | Load all M4 fixture files |
|
||||
| Test Case | Description |
|
||||
| ----------------------------------- | ------------------------------------------------- |
|
||||
| M4 Correction Revert Via CLI | Invoke plan correct --mode revert |
|
||||
| M4 Correction Append Via CLI | Invoke plan correct --mode append |
|
||||
| M4 Correction Dry Run Via CLI | Invoke plan correct --dry-run |
|
||||
| M4 Subplan Status Sequential | Plan status with sequential subplan config |
|
||||
| M4 Subplan Status Parallel | Plan status with parallel subplan config |
|
||||
| M4 Failure Handler Evaluation | SubplanFailureHandler decision logic |
|
||||
| M4 Fixture Loading | Load all M4 fixture files |
|
||||
| M4 Full Correction And Subplan Flow | End-to-end correction + subplan + failure handler |
|
||||
|
||||
Helper script: `robot/helper_m4_correction_subplan_smoke.py`
|
||||
@@ -1296,8 +1301,6 @@ nox -s benchmark
|
||||
SubplanFailureHandler logic, and fixture loading. Check
|
||||
`build/htmlcov/index.html` for uncovered lines in correction/plan CLI code.
|
||||
|
||||
---
|
||||
|
||||
## M5 ACMS Pipeline + Large-Project Context Smoke Tests
|
||||
|
||||
The M5 smoke suites verify the ACMS context pipeline foundation: context
|
||||
@@ -1308,16 +1311,16 @@ and project context policy resolution.
|
||||
|
||||
27 scenarios covering:
|
||||
|
||||
| Area | Scenarios |
|
||||
|------|-----------|
|
||||
| Fixture loading | Load ACMS context policy, large project context, analysis results |
|
||||
| Context policy resolution | Default view, inheritance chain, strategize/execute/apply overrides |
|
||||
| Budget enforcement | max_file_size, max_total_size, zero/negative validation, unlimited |
|
||||
| Context assembly (CLI) | list, add, show, clear via mocked ContextService |
|
||||
| Project context policy CLI | show policy, inspect/simulate stubs (NotImplementedError) |
|
||||
| Context analysis agent | Summary, dependencies, relevance scores (mocked) |
|
||||
| Multi-project context | Independent context per project |
|
||||
| Exclusion patterns | Glob-based path filtering |
|
||||
| Area | Scenarios |
|
||||
| -------------------------- | ------------------------------------------------------------------- |
|
||||
| Fixture loading | Load ACMS context policy, large project context, analysis results |
|
||||
| Context policy resolution | Default view, inheritance chain, strategize/execute/apply overrides |
|
||||
| Budget enforcement | max_file_size, max_total_size, zero/negative validation, unlimited |
|
||||
| Context assembly (CLI) | list, add, show, clear via mocked ContextService |
|
||||
| Project context policy CLI | show policy, inspect/simulate stubs (NotImplementedError) |
|
||||
| Context analysis agent | Summary, dependencies, relevance scores (mocked) |
|
||||
| Multi-project context | Independent context per project |
|
||||
| Exclusion patterns | Glob-based path filtering |
|
||||
|
||||
Step definitions: `features/steps/m5_acms_smoke_steps.py`
|
||||
Fixtures: `features/fixtures/m5/`
|
||||
@@ -1329,18 +1332,18 @@ Fixtures: `features/fixtures/m5/`
|
||||
|
||||
### Robot: `robot/m5_acms_smoke.robot`
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Load ACMS Context Policy Fixture | Validates fixture structure |
|
||||
| Load Large Project Context Fixture | Validates file count and entries |
|
||||
| Resolve Default View From Empty Policy | Empty policy includes all paths |
|
||||
| Resolve Strategize Inherits From Default | Phase inheritance works |
|
||||
| Resolve Strategize With Override | Override takes precedence |
|
||||
| Budget Max File Size Enforcement | File-level budget check |
|
||||
| Budget Max Total Size Enforcement | Aggregate budget check |
|
||||
| Invalid Phase Raises Error | Error handling for invalid phases |
|
||||
| Context Analysis Fixture Has Required Fields | Analysis result structure |
|
||||
| Multi Project Independent Context | Independent context per project |
|
||||
| Test Case | Description |
|
||||
| -------------------------------------------- | --------------------------------- |
|
||||
| Load ACMS Context Policy Fixture | Validates fixture structure |
|
||||
| Load Large Project Context Fixture | Validates file count and entries |
|
||||
| Resolve Default View From Empty Policy | Empty policy includes all paths |
|
||||
| Resolve Strategize Inherits From Default | Phase inheritance works |
|
||||
| Resolve Strategize With Override | Override takes precedence |
|
||||
| Budget Max File Size Enforcement | File-level budget check |
|
||||
| Budget Max Total Size Enforcement | Aggregate budget check |
|
||||
| Invalid Phase Raises Error | Error handling for invalid phases |
|
||||
| Context Analysis Fixture Has Required Fields | Analysis result structure |
|
||||
| Multi Project Independent Context | Independent context per project |
|
||||
|
||||
Helper script: `robot/helper_m5_acms_smoke.py`
|
||||
|
||||
@@ -1382,3 +1385,121 @@ nox -s benchmark
|
||||
- **Coverage drops**: The M5 smoke tests cover context policy resolution, budget
|
||||
enforcement, and CLI context commands. Check `build/htmlcov/index.html` for
|
||||
uncovered lines in context-related modules.
|
||||
|
||||
## M6 Autonomy Acceptance Suite
|
||||
|
||||
E2E fixtures covering ACP facade flows and autonomy guardrails. The suite
|
||||
exercises guardrail enforcement (max steps, tool budget, confirmations) and
|
||||
audit trail persistence in integrated scenarios.
|
||||
|
||||
### Overview
|
||||
|
||||
The M6 acceptance suite validates the autonomy hardening layer:
|
||||
|
||||
- **ACP Local Facade** — local-mode dispatch of all 11 supported operations
|
||||
(session, plan, registry, context, event).
|
||||
- **Autonomy Guardrails** — denylist/allowlist enforcement, cost budget caps,
|
||||
tool call limits, write approval gates, apply approval gates.
|
||||
- **Automation Profiles** — 8 built-in profiles, custom namespaced profiles,
|
||||
YAML loading, threshold validation, and four-level resolution precedence
|
||||
(plan > action > project > global).
|
||||
- **ACP Event Queue** — publish/subscribe, local callbacks, unsubscribe, close
|
||||
semantics, remote stub rejection.
|
||||
- **ACP HTTP Transport Stub** — all mutating methods raise
|
||||
`AcpNotAvailableError` in local mode.
|
||||
- **ACP Version Negotiation** — accept supported versions, reject unsupported.
|
||||
- **ACP Model Validation** — non-empty operation, valid status values,
|
||||
non-empty event types, non-empty error codes.
|
||||
|
||||
### Fixtures (`features/fixtures/m6/`)
|
||||
|
||||
| File | Purpose |
|
||||
| -------------------------- | ------------------------------------------------------------------------------- |
|
||||
| `acp_facade_flows.json` | ACP operation flows (session, plan, registry, context, event) |
|
||||
| `autonomy_guardrails.json` | Guard configurations (denylist, allowlist, budget, write/apply approval) |
|
||||
| `automation_profiles.json` | Built-in profile assertions, custom profile config, resolution precedence cases |
|
||||
|
||||
### Behave Suite: `features/m6_autonomy_acceptance.feature`
|
||||
|
||||
- **Fixture loading** — 3 scenarios verifying JSON structure
|
||||
- **ACP facade dispatch** — 10 scenarios covering all 11 operations
|
||||
- **ACP error handling** — 2 scenarios (unknown operation, invalid request type)
|
||||
- **Service registration** — 2 scenarios (register + list operations)
|
||||
- **Event queue** — 5 scenarios (publish, subscribe, unsubscribe, close, remote reject)
|
||||
- **HTTP transport stub** — 4 scenarios (send, connect, disconnect, is_connected)
|
||||
- **Version negotiation** — 3 scenarios (accept, reject, is_supported)
|
||||
- **Built-in profiles** — 3 scenarios (list, manual thresholds, full-auto thresholds)
|
||||
- **Profile creation** — 4 scenarios (custom, invalid name, bad threshold, YAML load)
|
||||
- **Guard enforcement** — 8 scenarios (denylist, allowlist, call limit, budget, write, apply, no guards)
|
||||
- **Profile resolution** — 3 scenarios (plan precedence, action precedence, global default)
|
||||
- **Service guard evaluation** — 1 scenario
|
||||
- **Model validation** — 4 scenarios (AcpRequest, AcpResponse, AcpEvent, AcpErrorDetail)
|
||||
|
||||
Step implementations: `features/steps/m6_autonomy_acceptance_steps.py`
|
||||
|
||||
### Robot Suite: `robot/m6_autonomy_acceptance.robot`
|
||||
|
||||
| Test Case | What it checks |
|
||||
| ------------------------------------- | -------------------------------------------------- |
|
||||
| M6 ACP Facade Session Lifecycle | session.create + session.close dispatch |
|
||||
| M6 ACP Facade Plan Lifecycle | plan create/execute/status/diff/apply |
|
||||
| M6 ACP Facade Unknown Operation Error | AcpOperationNotFoundError on bad op |
|
||||
| M6 ACP Event Queue Publish Subscribe | publish, subscribe, unsubscribe, close |
|
||||
| M6 ACP Transport Stub Rejects All | send/connect/disconnect raise AcpNotAvailableError |
|
||||
| M6 ACP Version Negotiation | accept 1.0, reject 2.0 |
|
||||
| M6 Guard Denylist Enforcement | denylist blocks denied tools |
|
||||
| M6 Guard Budget Enforcement | cost budget and call limit guards |
|
||||
| M6 Profile Resolution Precedence | plan > action > project > global |
|
||||
| M6 Fixture Loading | all 3 fixture files load and validate |
|
||||
| M6 Full Autonomy Acceptance Flow | end-to-end facade + guard + resolution |
|
||||
|
||||
Helper: `robot/helper_m6_autonomy_acceptance.py`
|
||||
|
||||
### ASV Benchmarks: `benchmarks/m6_acceptance_bench.py`
|
||||
|
||||
| Suite | Methods |
|
||||
| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `M6FacadeDispatchSuite` | `time_session_create`, `time_plan_create`, `time_plan_execute`, `time_list_operations` |
|
||||
| `M6GuardEvaluationSuite` | `time_denylist_check_allowed`, `time_denylist_check_denied`, `time_allowlist_check_allowed`, `time_allowlist_check_denied`, `time_budget_check_under`, `time_no_guards_check` |
|
||||
| `M6ProfileResolutionSuite` | `time_resolve_plan_level`, `time_resolve_action_level`, `time_resolve_global_default`, `time_get_builtin_profile`, `time_list_profiles`, `time_evaluate_guard`, `time_version_negotiation` |
|
||||
| `M6EventQueueSuite` | `time_publish_event`, `time_subscribe_and_publish`, `time_get_events` |
|
||||
| `M6FixtureLoadSuite` | `time_load_acp_facade_flows`, `time_load_autonomy_guardrails`, `time_load_automation_profiles`, `time_load_all_fixtures` |
|
||||
|
||||
### Running the M6 Autonomy Acceptance Suites
|
||||
|
||||
```bash
|
||||
# Behave only
|
||||
nox -s behave -- features/m6_autonomy_acceptance.feature
|
||||
|
||||
# Robot only
|
||||
nox -s robot -- robot/m6_autonomy_acceptance.robot
|
||||
|
||||
# ASV benchmarks
|
||||
nox -s benchmark
|
||||
|
||||
# Everything (default nox sessions)
|
||||
nox
|
||||
```
|
||||
|
||||
### Failure Triage Tips
|
||||
|
||||
- **`AmbiguousStep` errors**: All M6 smoke steps are prefixed with `m6 smoke`.
|
||||
If ambiguous, check that no other step file defines a conflicting pattern.
|
||||
- **Fixture file not found**: Verify `features/fixtures/m6/` contains all three
|
||||
fixture files (`acp_facade_flows.json`, `autonomy_guardrails.json`,
|
||||
`automation_profiles.json`).
|
||||
- **Robot `FAIL` sentinel**: Each Robot helper subcommand prints a detailed
|
||||
`FAIL:` line with exit code and output when something goes wrong.
|
||||
- **AcpNotAvailableError expected**: The transport stub and remote event
|
||||
subscribe are _supposed_ to raise this error. If a test fails here, it
|
||||
likely means the stub was changed to not raise.
|
||||
- **Guard evaluation failures**: Check that the `AutomationGuard` model fields
|
||||
match the fixture data. Denylist/allowlist/budget fields are all optional.
|
||||
- **Coverage drops**: The M6 acceptance tests cover the `acp/` package facade,
|
||||
events, transport, versioning, errors, and models, plus
|
||||
`automation_profile.py` and `automation_profile_service.py`. Check
|
||||
`build/htmlcov/index.html` for uncovered lines.
|
||||
|
||||
---
|
||||
|
||||
##
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"description": "ACP local-mode facade operation flows for M6 autonomy acceptance tests",
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "session_lifecycle",
|
||||
"operations": [
|
||||
{"operation": "session.create", "params": {}, "expected_status": "ok", "expected_keys": ["session_id", "status"]},
|
||||
{"operation": "session.close", "params": {}, "expected_status": "ok", "expected_keys": ["status"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "plan_lifecycle",
|
||||
"operations": [
|
||||
{"operation": "plan.create", "params": {}, "expected_status": "ok", "expected_keys": ["plan_id", "status"]},
|
||||
{"operation": "plan.execute", "params": {"plan_id": "01M6SM0KE00000000000000001"}, "expected_status": "ok", "expected_keys": ["plan_id", "status"]},
|
||||
{"operation": "plan.status", "params": {"plan_id": "01M6SM0KE00000000000000001"}, "expected_status": "ok", "expected_keys": ["plan_id", "phase"]},
|
||||
{"operation": "plan.diff", "params": {"plan_id": "01M6SM0KE00000000000000001"}, "expected_status": "ok", "expected_keys": ["plan_id", "changes"]},
|
||||
{"operation": "plan.apply", "params": {"plan_id": "01M6SM0KE00000000000000001"}, "expected_status": "ok", "expected_keys": ["plan_id", "status"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "registry_queries",
|
||||
"operations": [
|
||||
{"operation": "registry.list_tools", "params": {}, "expected_status": "ok", "expected_keys": ["tools"]},
|
||||
{"operation": "registry.list_resources", "params": {}, "expected_status": "ok", "expected_keys": ["resources"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "context_and_events",
|
||||
"operations": [
|
||||
{"operation": "context.get", "params": {}, "expected_status": "ok", "expected_keys": ["context"]},
|
||||
{"operation": "event.subscribe", "params": {}, "expected_status": "ok", "expected_keys": ["subscription_id", "status"]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unknown_operation",
|
||||
"operations": [
|
||||
{"operation": "nonexistent.op", "params": {}, "expected_error": "AcpOperationNotFoundError"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"description": "Automation profile configurations for M6 acceptance tests",
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "builtin_profiles",
|
||||
"expected_names": ["manual", "review", "supervised", "cautious", "trusted", "auto", "ci", "full-auto"],
|
||||
"manual_thresholds": {
|
||||
"auto_strategize": 1.0,
|
||||
"auto_execute": 1.0,
|
||||
"auto_apply": 1.0,
|
||||
"require_sandbox": true,
|
||||
"require_checkpoints": true,
|
||||
"allow_unsafe_tools": false
|
||||
},
|
||||
"full_auto_thresholds": {
|
||||
"auto_strategize": 0.0,
|
||||
"auto_execute": 0.0,
|
||||
"auto_apply": 0.0,
|
||||
"require_sandbox": false,
|
||||
"require_checkpoints": false,
|
||||
"allow_unsafe_tools": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "custom_profile",
|
||||
"config": {
|
||||
"name": "acme/strict",
|
||||
"description": "Strict profile for ACME Corp",
|
||||
"schema_version": "1.0",
|
||||
"auto_strategize": 1.0,
|
||||
"auto_execute": 1.0,
|
||||
"auto_apply": 1.0,
|
||||
"auto_decisions_strategize": 1.0,
|
||||
"auto_decisions_execute": 1.0,
|
||||
"auto_validation_fix": 1.0,
|
||||
"auto_strategy_revision": 1.0,
|
||||
"auto_reversion_from_apply": 1.0,
|
||||
"auto_child_plans": 1.0,
|
||||
"auto_retry_transient": 1.0,
|
||||
"auto_checkpoint_restore": 1.0,
|
||||
"require_sandbox": true,
|
||||
"require_checkpoints": true,
|
||||
"allow_unsafe_tools": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "profile_resolution_precedence",
|
||||
"cases": [
|
||||
{"plan": "ci", "action": "auto", "project": "manual", "expected": "ci"},
|
||||
{"plan": null, "action": "auto", "project": "manual", "expected": "auto"},
|
||||
{"plan": null, "action": null, "project": "manual", "expected": "manual"},
|
||||
{"plan": null, "action": null, "project": null, "expected": "manual"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"description": "Autonomy guardrail configurations for M6 acceptance tests",
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "denylist_guard",
|
||||
"profile": {
|
||||
"name": "test-denylist",
|
||||
"description": "Profile with tool denylist guard",
|
||||
"auto_strategize": 0.0,
|
||||
"auto_execute": 0.0,
|
||||
"auto_apply": 1.0,
|
||||
"guards": {
|
||||
"tool_denylist": ["rm_rf", "drop_database", "format_disk"],
|
||||
"require_approval_for_writes": false,
|
||||
"require_approval_for_apply": false
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{"tool": "read_file", "is_write": false, "expected_allowed": true},
|
||||
{"tool": "rm_rf", "is_write": false, "expected_allowed": false, "expected_reason": "denylist"},
|
||||
{"tool": "drop_database", "is_write": false, "expected_allowed": false, "expected_reason": "denylist"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "allowlist_guard",
|
||||
"profile": {
|
||||
"name": "test-allowlist",
|
||||
"description": "Profile with tool allowlist guard",
|
||||
"auto_strategize": 0.0,
|
||||
"auto_execute": 0.0,
|
||||
"auto_apply": 1.0,
|
||||
"guards": {
|
||||
"tool_allowlist": ["read_file", "list_dir", "search"],
|
||||
"require_approval_for_writes": false,
|
||||
"require_approval_for_apply": false
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{"tool": "read_file", "is_write": false, "expected_allowed": true},
|
||||
{"tool": "write_file", "is_write": false, "expected_allowed": false, "expected_reason": "allowlist"},
|
||||
{"tool": "search", "is_write": false, "expected_allowed": true}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cost_budget_guard",
|
||||
"profile": {
|
||||
"name": "test-budget",
|
||||
"description": "Profile with cost budget guard",
|
||||
"auto_strategize": 0.0,
|
||||
"auto_execute": 0.0,
|
||||
"auto_apply": 1.0,
|
||||
"guards": {
|
||||
"max_total_cost": 10.0,
|
||||
"max_tool_calls_per_step": 5
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{"tool": "llm_call", "cost_so_far": 3.0, "calls_so_far": 2, "expected_allowed": true},
|
||||
{"tool": "llm_call", "cost_so_far": 10.0, "calls_so_far": 2, "expected_allowed": false, "expected_reason": "budget"},
|
||||
{"tool": "llm_call", "cost_so_far": 3.0, "calls_so_far": 5, "expected_allowed": false, "expected_reason": "limit"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "write_approval_guard",
|
||||
"profile": {
|
||||
"name": "test-write-approval",
|
||||
"description": "Profile requiring write approval",
|
||||
"auto_strategize": 0.0,
|
||||
"auto_execute": 0.0,
|
||||
"auto_apply": 1.0,
|
||||
"guards": {
|
||||
"require_approval_for_writes": true,
|
||||
"require_approval_for_apply": true
|
||||
}
|
||||
},
|
||||
"checks": [
|
||||
{"tool": "read_file", "is_write": false, "expected_allowed": true},
|
||||
{"tool": "write_file", "is_write": true, "expected_allowed": false, "expected_reason": "Write operations"},
|
||||
{"tool": "__apply__", "is_write": false, "expected_allowed": false, "expected_reason": "Apply phase"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
Feature: M6 autonomy acceptance smoke tests
|
||||
As a developer working with the CleverAgents M6 milestone
|
||||
I want to verify ACP facade flows, autonomy guardrails, and audit trails
|
||||
So that the autonomy hardening layer enforces safety constraints correctly
|
||||
|
||||
Background:
|
||||
Given a m6 smoke test runner
|
||||
And a m6 smoke ACP local facade
|
||||
|
||||
# --- Fixture loading ---
|
||||
|
||||
Scenario: M6 smoke load ACP facade flows fixture
|
||||
When I m6 smoke load the ACP facade flows fixture
|
||||
Then the m6 smoke facade fixture should have a session lifecycle entry
|
||||
And the m6 smoke facade fixture should have a plan lifecycle entry
|
||||
|
||||
Scenario: M6 smoke load autonomy guardrails fixture
|
||||
When I m6 smoke load the autonomy guardrails fixture
|
||||
Then the m6 smoke guardrails fixture should have a denylist entry
|
||||
And the m6 smoke guardrails fixture should have an allowlist entry
|
||||
And the m6 smoke guardrails fixture should have a cost budget entry
|
||||
|
||||
Scenario: M6 smoke load automation profiles fixture
|
||||
When I m6 smoke load the automation profiles fixture
|
||||
Then the m6 smoke profiles fixture should list all 8 built-in names
|
||||
And the m6 smoke profiles fixture should have a custom profile entry
|
||||
|
||||
# --- ACP facade session operations ---
|
||||
|
||||
Scenario: M6 smoke ACP session create returns session id
|
||||
When I m6 smoke dispatch "session.create" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "session_id"
|
||||
And the m6 smoke response data should contain key "status"
|
||||
|
||||
Scenario: M6 smoke ACP session close returns closed status
|
||||
When I m6 smoke dispatch "session.close" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data "status" should equal "closed"
|
||||
|
||||
# --- ACP facade plan operations ---
|
||||
|
||||
Scenario: M6 smoke ACP plan create returns plan id
|
||||
When I m6 smoke dispatch "plan.create" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "plan_id"
|
||||
|
||||
Scenario: M6 smoke ACP plan execute queues plan
|
||||
When I m6 smoke dispatch "plan.execute" with params {"plan_id": "01M6SM0KE00000000000000001"}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data "status" should equal "queued"
|
||||
|
||||
Scenario: M6 smoke ACP plan status returns phase
|
||||
When I m6 smoke dispatch "plan.status" with params {"plan_id": "01M6SM0KE00000000000000001"}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "phase"
|
||||
|
||||
Scenario: M6 smoke ACP plan diff returns changes list
|
||||
When I m6 smoke dispatch "plan.diff" with params {"plan_id": "01M6SM0KE00000000000000001"}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "changes"
|
||||
|
||||
Scenario: M6 smoke ACP plan apply returns applied status
|
||||
When I m6 smoke dispatch "plan.apply" with params {"plan_id": "01M6SM0KE00000000000000001"}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data "status" should equal "applied"
|
||||
|
||||
# --- ACP facade registry and context operations ---
|
||||
|
||||
Scenario: M6 smoke ACP registry list tools returns empty list
|
||||
When I m6 smoke dispatch "registry.list_tools" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "tools"
|
||||
|
||||
Scenario: M6 smoke ACP registry list resources returns empty list
|
||||
When I m6 smoke dispatch "registry.list_resources" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "resources"
|
||||
|
||||
Scenario: M6 smoke ACP context get returns context dict
|
||||
When I m6 smoke dispatch "context.get" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "context"
|
||||
|
||||
Scenario: M6 smoke ACP event subscribe returns subscription id
|
||||
When I m6 smoke dispatch "event.subscribe" with params {}
|
||||
Then the m6 smoke response status should be "ok"
|
||||
And the m6 smoke response data should contain key "subscription_id"
|
||||
|
||||
# --- ACP facade error handling ---
|
||||
|
||||
Scenario: M6 smoke ACP unknown operation raises error
|
||||
When I m6 smoke dispatch unknown operation "nonexistent.op"
|
||||
Then the m6 smoke facade should raise AcpOperationNotFoundError
|
||||
|
||||
Scenario: M6 smoke ACP dispatch with invalid request type raises TypeError
|
||||
When I m6 smoke dispatch with a non-AcpRequest object
|
||||
Then the m6 smoke facade should raise TypeError
|
||||
|
||||
# --- ACP facade service registration ---
|
||||
|
||||
Scenario: M6 smoke ACP register service stores service
|
||||
When I m6 smoke register service "plan_service" on the facade
|
||||
Then the m6 smoke facade should have service "plan_service"
|
||||
|
||||
Scenario: M6 smoke ACP list operations returns all supported
|
||||
When I m6 smoke list facade operations
|
||||
Then the m6 smoke operations should include "session.create"
|
||||
And the m6 smoke operations should include "plan.execute"
|
||||
And the m6 smoke operations should include "event.subscribe"
|
||||
And the m6 smoke operations count should be 11
|
||||
|
||||
# --- ACP event queue ---
|
||||
|
||||
Scenario: M6 smoke ACP event queue publish and retrieve
|
||||
Given a m6 smoke ACP event queue
|
||||
When I m6 smoke publish an event with type "plan.progress"
|
||||
Then the m6 smoke event queue should have 1 event
|
||||
And the m6 smoke last event type should be "plan.progress"
|
||||
|
||||
Scenario: M6 smoke ACP event queue subscribe local callback
|
||||
Given a m6 smoke ACP event queue
|
||||
When I m6 smoke subscribe a local callback
|
||||
And I m6 smoke publish an event with type "plan.complete"
|
||||
Then the m6 smoke callback should have been called once
|
||||
|
||||
Scenario: M6 smoke ACP event queue unsubscribe
|
||||
Given a m6 smoke ACP event queue
|
||||
When I m6 smoke subscribe a local callback
|
||||
And I m6 smoke unsubscribe the callback
|
||||
And I m6 smoke publish an event with type "plan.complete"
|
||||
Then the m6 smoke callback should not have been called
|
||||
|
||||
Scenario: M6 smoke ACP event queue close prevents publish
|
||||
Given a m6 smoke ACP event queue
|
||||
When I m6 smoke close the event queue
|
||||
Then the m6 smoke publishing should raise RuntimeError
|
||||
|
||||
Scenario: M6 smoke ACP event queue remote subscribe raises error
|
||||
Given a m6 smoke ACP event queue
|
||||
When I m6 smoke attempt remote subscribe to "https://example.com/events"
|
||||
Then the m6 smoke facade should raise AcpNotAvailableError
|
||||
|
||||
# --- ACP HTTP transport stub ---
|
||||
|
||||
Scenario: M6 smoke ACP transport send raises not available
|
||||
When I m6 smoke attempt transport send
|
||||
Then the m6 smoke facade should raise AcpNotAvailableError
|
||||
|
||||
Scenario: M6 smoke ACP transport connect raises not available
|
||||
When I m6 smoke attempt transport connect to "https://example.com/acp"
|
||||
Then the m6 smoke facade should raise AcpNotAvailableError
|
||||
|
||||
Scenario: M6 smoke ACP transport disconnect raises not available
|
||||
When I m6 smoke attempt transport disconnect
|
||||
Then the m6 smoke facade should raise AcpNotAvailableError
|
||||
|
||||
Scenario: M6 smoke ACP transport is_connected returns false
|
||||
When I m6 smoke check transport is_connected
|
||||
Then the m6 smoke transport should not be connected
|
||||
|
||||
# --- ACP version negotiation ---
|
||||
|
||||
Scenario: M6 smoke ACP version negotiation accepts 1.0
|
||||
When I m6 smoke negotiate ACP version "1.0"
|
||||
Then the m6 smoke negotiated version should be "1.0"
|
||||
|
||||
Scenario: M6 smoke ACP version negotiation rejects unsupported
|
||||
When I m6 smoke negotiate ACP version "2.0"
|
||||
Then the m6 smoke facade should raise AcpVersionMismatchError
|
||||
|
||||
Scenario: M6 smoke ACP version is_supported returns correct result
|
||||
When I m6 smoke check if version "1.0" is supported
|
||||
Then the m6 smoke version support should be true
|
||||
When I m6 smoke check if version "99.0" is supported
|
||||
Then the m6 smoke version support should be false
|
||||
|
||||
# --- Automation profiles built-in ---
|
||||
|
||||
Scenario: M6 smoke all 8 built-in profiles exist
|
||||
When I m6 smoke list all built-in profiles
|
||||
Then the m6 smoke profile count should be 8
|
||||
And the m6 smoke profiles should include "manual"
|
||||
And the m6 smoke profiles should include "full-auto"
|
||||
|
||||
Scenario: M6 smoke manual profile has all thresholds at 1.0
|
||||
When I m6 smoke load built-in profile "manual"
|
||||
Then the m6 smoke profile auto_strategize should be 1.0
|
||||
And the m6 smoke profile auto_execute should be 1.0
|
||||
And the m6 smoke profile auto_apply should be 1.0
|
||||
And the m6 smoke profile require_sandbox should be true
|
||||
|
||||
Scenario: M6 smoke full-auto profile has no gates
|
||||
When I m6 smoke load built-in profile "full-auto"
|
||||
Then the m6 smoke profile auto_strategize should be 0.0
|
||||
And the m6 smoke profile auto_execute should be 0.0
|
||||
And the m6 smoke profile auto_apply should be 0.0
|
||||
And the m6 smoke profile require_sandbox should be false
|
||||
And the m6 smoke profile allow_unsafe_tools should be true
|
||||
|
||||
# --- Automation profile creation and validation ---
|
||||
|
||||
Scenario: M6 smoke create custom namespaced profile
|
||||
When I m6 smoke create a profile named "acme/strict" with auto_apply 1.0
|
||||
Then the m6 smoke created profile name should be "acme/strict"
|
||||
And the m6 smoke created profile auto_apply should be 1.0
|
||||
|
||||
Scenario: M6 smoke profile name validation rejects invalid
|
||||
When I m6 smoke create a profile with invalid name "has spaces"
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
|
||||
Scenario: M6 smoke profile threshold validation rejects out of range
|
||||
When I m6 smoke create a profile with auto_strategize 1.5
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
|
||||
Scenario: M6 smoke profile from_yaml loads correctly
|
||||
Given a m6 smoke temporary profile YAML file
|
||||
When I m6 smoke load profile from the temp YAML
|
||||
Then the m6 smoke loaded profile name should be "test-yaml-profile"
|
||||
|
||||
# --- Guard enforcement ---
|
||||
|
||||
Scenario: M6 smoke guard denylist blocks denied tool
|
||||
Given a m6 smoke profile with denylist guard for "rm_rf"
|
||||
When I m6 smoke check guard for tool "rm_rf"
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "denylist"
|
||||
|
||||
Scenario: M6 smoke guard denylist allows non-denied tool
|
||||
Given a m6 smoke profile with denylist guard for "rm_rf"
|
||||
When I m6 smoke check guard for tool "read_file"
|
||||
Then the m6 smoke guard result should be allowed
|
||||
|
||||
Scenario: M6 smoke guard allowlist blocks unlisted tool
|
||||
Given a m6 smoke profile with allowlist guard for "read_file" and "search"
|
||||
When I m6 smoke check guard for tool "write_file"
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "allowlist"
|
||||
|
||||
Scenario: M6 smoke guard max tool calls blocks at limit
|
||||
Given a m6 smoke profile with max 5 tool calls per step
|
||||
When I m6 smoke check guard for tool "llm_call" with 5 calls so far
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "limit"
|
||||
|
||||
Scenario: M6 smoke guard cost budget blocks at cap
|
||||
Given a m6 smoke profile with max cost 10.0
|
||||
When I m6 smoke check guard for tool "llm_call" with cost 10.0
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "budget"
|
||||
|
||||
Scenario: M6 smoke guard write approval blocks write operations
|
||||
Given a m6 smoke profile with write approval required
|
||||
When I m6 smoke check guard for tool "write_file" as a write operation
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "Write operations"
|
||||
|
||||
Scenario: M6 smoke guard apply approval blocks apply phase
|
||||
Given a m6 smoke profile with apply approval required
|
||||
When I m6 smoke check guard for tool "__apply__"
|
||||
Then the m6 smoke guard result should not be allowed
|
||||
And the m6 smoke guard reason should contain "Apply phase"
|
||||
|
||||
Scenario: M6 smoke guard with no guards allows everything
|
||||
Given a m6 smoke profile with no guards
|
||||
When I m6 smoke check guard for tool "anything"
|
||||
Then the m6 smoke guard result should be allowed
|
||||
|
||||
# --- Profile service resolution precedence ---
|
||||
|
||||
Scenario: M6 smoke profile resolution plan takes precedence
|
||||
Given a m6 smoke automation profile service
|
||||
When I m6 smoke resolve profile with plan "ci" action "auto" project "manual"
|
||||
Then the m6 smoke resolved profile name should be "ci"
|
||||
|
||||
Scenario: M6 smoke profile resolution action takes precedence over project
|
||||
Given a m6 smoke automation profile service
|
||||
When I m6 smoke resolve profile with plan null action "auto" project "manual"
|
||||
Then the m6 smoke resolved profile name should be "auto"
|
||||
|
||||
Scenario: M6 smoke profile resolution falls back to global default
|
||||
Given a m6 smoke automation profile service
|
||||
When I m6 smoke resolve profile with plan null action null project null
|
||||
Then the m6 smoke resolved profile name should be "manual"
|
||||
|
||||
# --- Profile service guard evaluation ---
|
||||
|
||||
Scenario: M6 smoke service evaluate guard delegates to profile
|
||||
Given a m6 smoke automation profile service
|
||||
When I m6 smoke evaluate guard for profile "manual" and tool "read_file"
|
||||
Then the m6 smoke guard result should be allowed
|
||||
|
||||
# --- ACP model validation ---
|
||||
|
||||
Scenario: M6 smoke AcpRequest validates non-empty operation
|
||||
When I m6 smoke create AcpRequest with empty operation
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
|
||||
Scenario: M6 smoke AcpResponse validates status values
|
||||
When I m6 smoke create AcpResponse with invalid status "maybe"
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
|
||||
Scenario: M6 smoke AcpEvent validates non-empty event_type
|
||||
When I m6 smoke create AcpEvent with empty event_type
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
|
||||
Scenario: M6 smoke AcpErrorDetail validates non-empty fields
|
||||
When I m6 smoke create AcpErrorDetail with empty code
|
||||
Then the m6 smoke creation should raise ValueError
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Step definitions for M6 ACP facade, event queue, transport, and version tests.
|
||||
|
||||
Split from ``m6_autonomy_acceptance_steps.py`` to stay under the project's
|
||||
500-line guideline. All step names keep the ``m6 smoke`` prefix to avoid
|
||||
``AmbiguousStep`` conflicts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.acp.errors import (
|
||||
AcpNotAvailableError,
|
||||
AcpOperationNotFoundError,
|
||||
AcpVersionMismatchError,
|
||||
)
|
||||
from cleveragents.acp.events import AcpEventQueue
|
||||
from cleveragents.acp.facade import AcpLocalFacade
|
||||
from cleveragents.acp.models import (
|
||||
AcpErrorDetail,
|
||||
AcpEvent,
|
||||
AcpRequest,
|
||||
AcpResponse,
|
||||
)
|
||||
from cleveragents.acp.transport import AcpHttpTransport
|
||||
from cleveragents.acp.versioning import AcpVersionNegotiator
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m6"
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Background
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m6 smoke test runner")
|
||||
def step_m6_smoke_runner(context: Context) -> None:
|
||||
"""Initialise the m6 smoke test context."""
|
||||
context.m6_result = None
|
||||
context.m6_response = None
|
||||
context.m6_error = None
|
||||
context.m6_profile = None
|
||||
context.m6_guard_result = None
|
||||
context.m6_operations = None
|
||||
context.m6_event_queue = None
|
||||
context.m6_callback_calls = []
|
||||
context.m6_subscription_id = None
|
||||
context.m6_version_result = None
|
||||
context.m6_version_supported = None
|
||||
context.m6_transport_connected = None
|
||||
context.m6_profiles_list = []
|
||||
|
||||
|
||||
@given("a m6 smoke ACP local facade")
|
||||
def step_m6_smoke_facade(context: Context) -> None:
|
||||
"""Create an AcpLocalFacade for testing."""
|
||||
context.m6_facade = AcpLocalFacade()
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Fixture loading — ACP facade flows
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m6 smoke load the ACP facade flows fixture")
|
||||
def step_m6_smoke_load_facade_fixture(context: Context) -> None:
|
||||
with open(_FIXTURES_DIR / "acp_facade_flows.json") as f:
|
||||
context.m6_fixture_data = json.load(f)
|
||||
|
||||
|
||||
@then("the m6 smoke facade fixture should have a session lifecycle entry")
|
||||
def step_m6_smoke_facade_session(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_fixture_data["fixtures"]]
|
||||
assert "session_lifecycle" in names
|
||||
|
||||
|
||||
@then("the m6 smoke facade fixture should have a plan lifecycle entry")
|
||||
def step_m6_smoke_facade_plan(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_fixture_data["fixtures"]]
|
||||
assert "plan_lifecycle" in names
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP facade dispatch operations
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke dispatch "{operation}" with params {params_json}')
|
||||
def step_m6_smoke_dispatch(
|
||||
context: Context,
|
||||
operation: str,
|
||||
params_json: str,
|
||||
) -> None:
|
||||
params = json.loads(params_json)
|
||||
request = AcpRequest(operation=operation, params=params)
|
||||
context.m6_response = context.m6_facade.dispatch(request)
|
||||
|
||||
|
||||
@then('the m6 smoke response status should be "{status}"')
|
||||
def step_m6_smoke_response_status(context: Context, status: str) -> None:
|
||||
assert context.m6_response is not None
|
||||
assert context.m6_response.status == status
|
||||
|
||||
|
||||
@then('the m6 smoke response data should contain key "{key}"')
|
||||
def step_m6_smoke_response_key(context: Context, key: str) -> None:
|
||||
assert context.m6_response is not None
|
||||
assert key in context.m6_response.data
|
||||
|
||||
|
||||
@then('the m6 smoke response data "{key}" should equal "{value}"')
|
||||
def step_m6_smoke_response_value(
|
||||
context: Context,
|
||||
key: str,
|
||||
value: str,
|
||||
) -> None:
|
||||
assert context.m6_response is not None
|
||||
assert context.m6_response.data[key] == value
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP facade error handling
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke dispatch unknown operation "{operation}"')
|
||||
def step_m6_smoke_dispatch_unknown(context: Context, operation: str) -> None:
|
||||
request = AcpRequest(operation=operation, params={})
|
||||
try:
|
||||
context.m6_facade.dispatch(request)
|
||||
context.m6_error = None
|
||||
except AcpOperationNotFoundError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@then("the m6 smoke facade should raise AcpOperationNotFoundError")
|
||||
def step_m6_smoke_error_op_not_found(context: Context) -> None:
|
||||
assert isinstance(context.m6_error, AcpOperationNotFoundError)
|
||||
|
||||
|
||||
@when("I m6 smoke dispatch with a non-AcpRequest object")
|
||||
def step_m6_smoke_dispatch_invalid(context: Context) -> None:
|
||||
try:
|
||||
context.m6_facade.dispatch("not a request") # type: ignore[arg-type]
|
||||
context.m6_error = None
|
||||
except TypeError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@then("the m6 smoke facade should raise TypeError")
|
||||
def step_m6_smoke_error_type(context: Context) -> None:
|
||||
assert isinstance(context.m6_error, TypeError)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP facade service registration and list operations
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke register service "{name}" on the facade')
|
||||
def step_m6_smoke_register_service(context: Context, name: str) -> None:
|
||||
mock_service = MagicMock()
|
||||
context.m6_facade.register_service(name, mock_service)
|
||||
|
||||
|
||||
@then('the m6 smoke facade should have service "{name}"')
|
||||
def step_m6_smoke_has_service(context: Context, name: str) -> None:
|
||||
assert name in context.m6_facade._services
|
||||
|
||||
|
||||
@when("I m6 smoke list facade operations")
|
||||
def step_m6_smoke_list_ops(context: Context) -> None:
|
||||
context.m6_operations = context.m6_facade.list_operations()
|
||||
|
||||
|
||||
@then('the m6 smoke operations should include "{operation}"')
|
||||
def step_m6_smoke_ops_include(context: Context, operation: str) -> None:
|
||||
assert operation in context.m6_operations
|
||||
|
||||
|
||||
@then("the m6 smoke operations count should be {count:d}")
|
||||
def step_m6_smoke_ops_count(context: Context, count: int) -> None:
|
||||
assert len(context.m6_operations) == count
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP event queue
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m6 smoke ACP event queue")
|
||||
def step_m6_smoke_event_queue(context: Context) -> None:
|
||||
context.m6_event_queue = AcpEventQueue()
|
||||
context.m6_callback_calls = []
|
||||
|
||||
|
||||
@when('I m6 smoke publish an event with type "{event_type}"')
|
||||
def step_m6_smoke_publish_event(context: Context, event_type: str) -> None:
|
||||
event = AcpEvent(event_type=event_type, data={"test": True})
|
||||
context.m6_event_queue.publish(event)
|
||||
|
||||
|
||||
@then("the m6 smoke event queue should have {count:d} event")
|
||||
def step_m6_smoke_event_count(context: Context, count: int) -> None:
|
||||
events = context.m6_event_queue.get_events()
|
||||
assert len(events) == count
|
||||
|
||||
|
||||
@then('the m6 smoke last event type should be "{event_type}"')
|
||||
def step_m6_smoke_last_event_type(context: Context, event_type: str) -> None:
|
||||
events = context.m6_event_queue.get_events()
|
||||
assert events[-1].event_type == event_type
|
||||
|
||||
|
||||
@when("I m6 smoke subscribe a local callback")
|
||||
def step_m6_smoke_subscribe_local(context: Context) -> None:
|
||||
def _callback(event: AcpEvent) -> None:
|
||||
context.m6_callback_calls.append(event)
|
||||
|
||||
context.m6_subscription_id = context.m6_event_queue.subscribe_local(_callback)
|
||||
|
||||
|
||||
@then("the m6 smoke callback should have been called once")
|
||||
def step_m6_smoke_callback_called(context: Context) -> None:
|
||||
assert len(context.m6_callback_calls) == 1
|
||||
|
||||
|
||||
@when("I m6 smoke unsubscribe the callback")
|
||||
def step_m6_smoke_unsubscribe(context: Context) -> None:
|
||||
context.m6_event_queue.unsubscribe(context.m6_subscription_id)
|
||||
|
||||
|
||||
@then("the m6 smoke callback should not have been called")
|
||||
def step_m6_smoke_callback_not_called(context: Context) -> None:
|
||||
assert len(context.m6_callback_calls) == 0
|
||||
|
||||
|
||||
@when("I m6 smoke close the event queue")
|
||||
def step_m6_smoke_close_queue(context: Context) -> None:
|
||||
context.m6_event_queue.close()
|
||||
|
||||
|
||||
@then("the m6 smoke publishing should raise RuntimeError")
|
||||
def step_m6_smoke_publish_after_close(context: Context) -> None:
|
||||
event = AcpEvent(event_type="after.close", data={})
|
||||
try:
|
||||
context.m6_event_queue.publish(event)
|
||||
raise AssertionError("Expected RuntimeError")
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
@when('I m6 smoke attempt remote subscribe to "{endpoint}"')
|
||||
def step_m6_smoke_remote_subscribe(context: Context, endpoint: str) -> None:
|
||||
try:
|
||||
context.m6_event_queue.subscribe_remote(endpoint)
|
||||
context.m6_error = None
|
||||
except AcpNotAvailableError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@then("the m6 smoke facade should raise AcpNotAvailableError")
|
||||
def step_m6_smoke_error_not_available(context: Context) -> None:
|
||||
assert isinstance(context.m6_error, AcpNotAvailableError)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP HTTP transport stub
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m6 smoke attempt transport send")
|
||||
def step_m6_smoke_transport_send(context: Context) -> None:
|
||||
transport = AcpHttpTransport()
|
||||
request = AcpRequest(operation="plan.create", params={})
|
||||
try:
|
||||
transport.send(request)
|
||||
context.m6_error = None
|
||||
except AcpNotAvailableError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when('I m6 smoke attempt transport connect to "{url}"')
|
||||
def step_m6_smoke_transport_connect(context: Context, url: str) -> None:
|
||||
transport = AcpHttpTransport()
|
||||
try:
|
||||
transport.connect(url)
|
||||
context.m6_error = None
|
||||
except AcpNotAvailableError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when("I m6 smoke attempt transport disconnect")
|
||||
def step_m6_smoke_transport_disconnect(context: Context) -> None:
|
||||
transport = AcpHttpTransport()
|
||||
try:
|
||||
transport.disconnect()
|
||||
context.m6_error = None
|
||||
except AcpNotAvailableError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when("I m6 smoke check transport is_connected")
|
||||
def step_m6_smoke_transport_connected(context: Context) -> None:
|
||||
transport = AcpHttpTransport()
|
||||
context.m6_transport_connected = transport.is_connected()
|
||||
|
||||
|
||||
@then("the m6 smoke transport should not be connected")
|
||||
def step_m6_smoke_transport_not_connected(context: Context) -> None:
|
||||
assert context.m6_transport_connected is False
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP version negotiation
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke negotiate ACP version "{version}"')
|
||||
def step_m6_smoke_negotiate(context: Context, version: str) -> None:
|
||||
negotiator = AcpVersionNegotiator()
|
||||
try:
|
||||
context.m6_version_result = negotiator.negotiate(version)
|
||||
context.m6_error = None
|
||||
except AcpVersionMismatchError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@then('the m6 smoke negotiated version should be "{version}"')
|
||||
def step_m6_smoke_negotiated(context: Context, version: str) -> None:
|
||||
assert context.m6_version_result == version
|
||||
|
||||
|
||||
@then("the m6 smoke facade should raise AcpVersionMismatchError")
|
||||
def step_m6_smoke_error_version(context: Context) -> None:
|
||||
assert isinstance(context.m6_error, AcpVersionMismatchError)
|
||||
|
||||
|
||||
@when('I m6 smoke check if version "{version}" is supported')
|
||||
def step_m6_smoke_version_supported(context: Context, version: str) -> None:
|
||||
negotiator = AcpVersionNegotiator()
|
||||
context.m6_version_supported = negotiator.is_supported(version)
|
||||
|
||||
|
||||
@then("the m6 smoke version support should be true")
|
||||
def step_m6_smoke_version_true(context: Context) -> None:
|
||||
assert context.m6_version_supported is True
|
||||
|
||||
|
||||
@then("the m6 smoke version support should be false")
|
||||
def step_m6_smoke_version_false(context: Context) -> None:
|
||||
assert context.m6_version_supported is False
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# ACP model validation
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m6 smoke create AcpRequest with empty operation")
|
||||
def step_m6_smoke_invalid_request(context: Context) -> None:
|
||||
try:
|
||||
AcpRequest(operation="")
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when('I m6 smoke create AcpResponse with invalid status "{status}"')
|
||||
def step_m6_smoke_invalid_response(context: Context, status: str) -> None:
|
||||
try:
|
||||
AcpResponse(request_id="test", status=status)
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when("I m6 smoke create AcpEvent with empty event_type")
|
||||
def step_m6_smoke_invalid_event(context: Context) -> None:
|
||||
try:
|
||||
AcpEvent(event_type="")
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@when("I m6 smoke create AcpErrorDetail with empty code")
|
||||
def step_m6_smoke_invalid_error_detail(context: Context) -> None:
|
||||
try:
|
||||
AcpErrorDetail(code="", message="test")
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Step definitions for M6 autonomy guardrails, profiles, and guard enforcement.
|
||||
|
||||
Split from ``m6_autonomy_acceptance_steps.py`` to stay under the project's
|
||||
500-line guideline. All step names keep the ``m6 smoke`` prefix to avoid
|
||||
``AmbiguousStep`` conflicts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.application.services.automation_profile_service import (
|
||||
AutomationProfileService,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_guard import (
|
||||
AutomationGuard,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import (
|
||||
BUILTIN_PROFILES,
|
||||
AutomationProfile,
|
||||
)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m6"
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Fixture loading — guardrails and profiles
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m6 smoke load the autonomy guardrails fixture")
|
||||
def step_m6_smoke_load_guardrails(context: Context) -> None:
|
||||
with open(_FIXTURES_DIR / "autonomy_guardrails.json") as f:
|
||||
context.m6_guardrails_data = json.load(f)
|
||||
|
||||
|
||||
@then("the m6 smoke guardrails fixture should have a denylist entry")
|
||||
def step_m6_smoke_guardrails_denylist(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_guardrails_data["fixtures"]]
|
||||
assert "denylist_guard" in names
|
||||
|
||||
|
||||
@then("the m6 smoke guardrails fixture should have an allowlist entry")
|
||||
def step_m6_smoke_guardrails_allowlist(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_guardrails_data["fixtures"]]
|
||||
assert "allowlist_guard" in names
|
||||
|
||||
|
||||
@then("the m6 smoke guardrails fixture should have a cost budget entry")
|
||||
def step_m6_smoke_guardrails_budget(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_guardrails_data["fixtures"]]
|
||||
assert "cost_budget_guard" in names
|
||||
|
||||
|
||||
@when("I m6 smoke load the automation profiles fixture")
|
||||
def step_m6_smoke_load_profiles_fixture(context: Context) -> None:
|
||||
with open(_FIXTURES_DIR / "automation_profiles.json") as f:
|
||||
context.m6_profiles_data = json.load(f)
|
||||
|
||||
|
||||
@then("the m6 smoke profiles fixture should list all 8 built-in names")
|
||||
def step_m6_smoke_profiles_builtin_count(context: Context) -> None:
|
||||
entry = next(
|
||||
f
|
||||
for f in context.m6_profiles_data["fixtures"]
|
||||
if f["name"] == "builtin_profiles"
|
||||
)
|
||||
assert len(entry["expected_names"]) == 8
|
||||
|
||||
|
||||
@then("the m6 smoke profiles fixture should have a custom profile entry")
|
||||
def step_m6_smoke_profiles_custom(context: Context) -> None:
|
||||
names = [f["name"] for f in context.m6_profiles_data["fixtures"]]
|
||||
assert "custom_profile" in names
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Automation profiles built-in
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m6 smoke list all built-in profiles")
|
||||
def step_m6_smoke_list_builtins(context: Context) -> None:
|
||||
context.m6_profiles_list = list(BUILTIN_PROFILES.keys())
|
||||
|
||||
|
||||
@then("the m6 smoke profile count should be {count:d}")
|
||||
def step_m6_smoke_profile_count(context: Context, count: int) -> None:
|
||||
assert len(context.m6_profiles_list) == count
|
||||
|
||||
|
||||
@then('the m6 smoke profiles should include "{name}"')
|
||||
def step_m6_smoke_profiles_include(context: Context, name: str) -> None:
|
||||
assert name in context.m6_profiles_list
|
||||
|
||||
|
||||
@when('I m6 smoke load built-in profile "{name}"')
|
||||
def step_m6_smoke_load_builtin(context: Context, name: str) -> None:
|
||||
context.m6_profile = BUILTIN_PROFILES[name]
|
||||
|
||||
|
||||
@then("the m6 smoke profile auto_strategize should be {value:g}")
|
||||
def step_m6_smoke_profile_auto_strategize(
|
||||
context: Context,
|
||||
value: float,
|
||||
) -> None:
|
||||
assert context.m6_profile.auto_strategize == value
|
||||
|
||||
|
||||
@then("the m6 smoke profile auto_execute should be {value:g}")
|
||||
def step_m6_smoke_profile_auto_execute(
|
||||
context: Context,
|
||||
value: float,
|
||||
) -> None:
|
||||
assert context.m6_profile.auto_execute == value
|
||||
|
||||
|
||||
@then("the m6 smoke profile auto_apply should be {value:g}")
|
||||
def step_m6_smoke_profile_auto_apply(
|
||||
context: Context,
|
||||
value: float,
|
||||
) -> None:
|
||||
assert context.m6_profile.auto_apply == value
|
||||
|
||||
|
||||
@then("the m6 smoke profile require_sandbox should be true")
|
||||
def step_m6_smoke_profile_sandbox_true(context: Context) -> None:
|
||||
assert context.m6_profile.require_sandbox is True
|
||||
|
||||
|
||||
@then("the m6 smoke profile require_sandbox should be false")
|
||||
def step_m6_smoke_profile_sandbox_false(context: Context) -> None:
|
||||
assert context.m6_profile.require_sandbox is False
|
||||
|
||||
|
||||
@then("the m6 smoke profile allow_unsafe_tools should be true")
|
||||
def step_m6_smoke_profile_unsafe_true(context: Context) -> None:
|
||||
assert context.m6_profile.allow_unsafe_tools is True
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Automation profile creation and validation
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke create a profile named "{name}" with auto_apply {value:g}')
|
||||
def step_m6_smoke_create_profile(
|
||||
context: Context,
|
||||
name: str,
|
||||
value: float,
|
||||
) -> None:
|
||||
context.m6_profile = AutomationProfile(name=name, auto_apply=value)
|
||||
|
||||
|
||||
@then('the m6 smoke created profile name should be "{name}"')
|
||||
def step_m6_smoke_created_name(context: Context, name: str) -> None:
|
||||
assert context.m6_profile.name == name
|
||||
|
||||
|
||||
@then("the m6 smoke created profile auto_apply should be {value:g}")
|
||||
def step_m6_smoke_created_auto_apply(context: Context, value: float) -> None:
|
||||
assert context.m6_profile.auto_apply == value
|
||||
|
||||
|
||||
@when('I m6 smoke create a profile with invalid name "{name}"')
|
||||
def step_m6_smoke_invalid_name(context: Context, name: str) -> None:
|
||||
try:
|
||||
AutomationProfile(name=name)
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@then("the m6 smoke creation should raise ValueError")
|
||||
def step_m6_smoke_error_value(context: Context) -> None:
|
||||
assert isinstance(context.m6_error, (ValueError,))
|
||||
|
||||
|
||||
@when("I m6 smoke create a profile with auto_strategize {value:g}")
|
||||
def step_m6_smoke_invalid_threshold(context: Context, value: float) -> None:
|
||||
try:
|
||||
AutomationProfile(name="test-bad-threshold", auto_strategize=value)
|
||||
context.m6_error = None
|
||||
except ValueError as exc:
|
||||
context.m6_error = exc
|
||||
|
||||
|
||||
@given("a m6 smoke temporary profile YAML file")
|
||||
def step_m6_smoke_yaml_file(context: Context) -> None:
|
||||
config = {
|
||||
"name": "test-yaml-profile",
|
||||
"description": "A profile loaded from YAML",
|
||||
"auto_strategize": 0.5,
|
||||
"auto_execute": 0.5,
|
||||
"auto_apply": 1.0,
|
||||
}
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
suffix=".yaml",
|
||||
delete=False,
|
||||
) as tmp:
|
||||
yaml.dump(config, tmp)
|
||||
context.m6_yaml_path = tmp.name
|
||||
|
||||
|
||||
@when("I m6 smoke load profile from the temp YAML")
|
||||
def step_m6_smoke_load_yaml(context: Context) -> None:
|
||||
context.m6_profile = AutomationProfile.from_yaml(context.m6_yaml_path)
|
||||
|
||||
|
||||
@then('the m6 smoke loaded profile name should be "{name}"')
|
||||
def step_m6_smoke_loaded_name(context: Context, name: str) -> None:
|
||||
assert context.m6_profile.name == name
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Guard enforcement
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a m6 smoke profile with denylist guard for "{tool}"')
|
||||
def step_m6_smoke_denylist_guard(context: Context, tool: str) -> None:
|
||||
guard = AutomationGuard(tool_denylist=[tool])
|
||||
context.m6_profile = AutomationProfile(name="test-deny", guards=guard)
|
||||
|
||||
|
||||
@when('I m6 smoke check guard for tool "{tool}"')
|
||||
def step_m6_smoke_check_guard(context: Context, tool: str) -> None:
|
||||
context.m6_guard_result = context.m6_profile.check_guard(tool_name=tool)
|
||||
|
||||
|
||||
@then("the m6 smoke guard result should not be allowed")
|
||||
def step_m6_smoke_guard_not_allowed(context: Context) -> None:
|
||||
assert context.m6_guard_result.allowed is False
|
||||
|
||||
|
||||
@then("the m6 smoke guard result should be allowed")
|
||||
def step_m6_smoke_guard_allowed(context: Context) -> None:
|
||||
assert context.m6_guard_result.allowed is True
|
||||
|
||||
|
||||
@then('the m6 smoke guard reason should contain "{text}"')
|
||||
def step_m6_smoke_guard_reason(context: Context, text: str) -> None:
|
||||
assert context.m6_guard_result.reason is not None
|
||||
assert text in context.m6_guard_result.reason
|
||||
|
||||
|
||||
@given('a m6 smoke profile with allowlist guard for "{t1}" and "{t2}"')
|
||||
def step_m6_smoke_allowlist_guard(
|
||||
context: Context,
|
||||
t1: str,
|
||||
t2: str,
|
||||
) -> None:
|
||||
guard = AutomationGuard(tool_allowlist=[t1, t2])
|
||||
context.m6_profile = AutomationProfile(name="test-allow", guards=guard)
|
||||
|
||||
|
||||
@given("a m6 smoke profile with max {limit:d} tool calls per step")
|
||||
def step_m6_smoke_max_calls_guard(context: Context, limit: int) -> None:
|
||||
guard = AutomationGuard(max_tool_calls_per_step=limit)
|
||||
context.m6_profile = AutomationProfile(name="test-maxcall", guards=guard)
|
||||
|
||||
|
||||
@when('I m6 smoke check guard for tool "{tool}" with {calls:d} calls so far')
|
||||
def step_m6_smoke_check_guard_calls(
|
||||
context: Context,
|
||||
tool: str,
|
||||
calls: int,
|
||||
) -> None:
|
||||
context.m6_guard_result = context.m6_profile.check_guard(
|
||||
tool_name=tool,
|
||||
calls_so_far=calls,
|
||||
)
|
||||
|
||||
|
||||
@given("a m6 smoke profile with max cost {budget:g}")
|
||||
def step_m6_smoke_cost_guard(context: Context, budget: float) -> None:
|
||||
guard = AutomationGuard(max_total_cost=budget)
|
||||
context.m6_profile = AutomationProfile(name="test-budget", guards=guard)
|
||||
|
||||
|
||||
@when('I m6 smoke check guard for tool "{tool}" with cost {cost:g}')
|
||||
def step_m6_smoke_check_guard_cost(
|
||||
context: Context,
|
||||
tool: str,
|
||||
cost: float,
|
||||
) -> None:
|
||||
context.m6_guard_result = context.m6_profile.check_guard(
|
||||
tool_name=tool,
|
||||
cost_so_far=cost,
|
||||
)
|
||||
|
||||
|
||||
@given("a m6 smoke profile with write approval required")
|
||||
def step_m6_smoke_write_guard(context: Context) -> None:
|
||||
guard = AutomationGuard(require_approval_for_writes=True)
|
||||
context.m6_profile = AutomationProfile(
|
||||
name="test-writeapproval",
|
||||
guards=guard,
|
||||
)
|
||||
|
||||
|
||||
@when('I m6 smoke check guard for tool "{tool}" as a write operation')
|
||||
def step_m6_smoke_check_guard_write(context: Context, tool: str) -> None:
|
||||
context.m6_guard_result = context.m6_profile.check_guard(
|
||||
tool_name=tool,
|
||||
is_write=True,
|
||||
)
|
||||
|
||||
|
||||
@given("a m6 smoke profile with apply approval required")
|
||||
def step_m6_smoke_apply_guard(context: Context) -> None:
|
||||
guard = AutomationGuard(require_approval_for_apply=True)
|
||||
context.m6_profile = AutomationProfile(
|
||||
name="test-applyapproval",
|
||||
guards=guard,
|
||||
)
|
||||
|
||||
|
||||
@given("a m6 smoke profile with no guards")
|
||||
def step_m6_smoke_no_guards(context: Context) -> None:
|
||||
context.m6_profile = AutomationProfile(name="test-noguards")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Profile service resolution
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m6 smoke automation profile service")
|
||||
def step_m6_smoke_profile_service(context: Context) -> None:
|
||||
context.m6_profile_service = AutomationProfileService()
|
||||
|
||||
|
||||
@when(
|
||||
"I m6 smoke resolve profile with plan "
|
||||
'"{plan}" action "{action}" project "{project}"'
|
||||
)
|
||||
def step_m6_smoke_resolve(
|
||||
context: Context,
|
||||
plan: str,
|
||||
action: str,
|
||||
project: str,
|
||||
) -> None:
|
||||
context.m6_profile = context.m6_profile_service.resolve_profile(
|
||||
plan_profile=plan if plan != "null" else None,
|
||||
action_profile=action if action != "null" else None,
|
||||
project_profile=project if project != "null" else None,
|
||||
)
|
||||
|
||||
|
||||
@when('I m6 smoke resolve profile with plan null action "{action}" project "{project}"')
|
||||
def step_m6_smoke_resolve_no_plan(
|
||||
context: Context,
|
||||
action: str,
|
||||
project: str,
|
||||
) -> None:
|
||||
context.m6_profile = context.m6_profile_service.resolve_profile(
|
||||
plan_profile=None,
|
||||
action_profile=action if action != "null" else None,
|
||||
project_profile=project if project != "null" else None,
|
||||
)
|
||||
|
||||
|
||||
@when("I m6 smoke resolve profile with plan null action null project null")
|
||||
def step_m6_smoke_resolve_all_null(context: Context) -> None:
|
||||
context.m6_profile = context.m6_profile_service.resolve_profile(
|
||||
plan_profile=None,
|
||||
action_profile=None,
|
||||
project_profile=None,
|
||||
)
|
||||
|
||||
|
||||
@then('the m6 smoke resolved profile name should be "{name}"')
|
||||
def step_m6_smoke_resolved_name(context: Context, name: str) -> None:
|
||||
assert context.m6_profile.name == name
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Profile service guard evaluation
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m6 smoke evaluate guard for profile "{profile}" and tool "{tool}"')
|
||||
def step_m6_smoke_evaluate_guard(
|
||||
context: Context,
|
||||
profile: str,
|
||||
tool: str,
|
||||
) -> None:
|
||||
context.m6_guard_result = context.m6_profile_service.evaluate_guard(
|
||||
profile_name=profile,
|
||||
tool_name=tool,
|
||||
)
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Helper script for m6_autonomy_acceptance.robot E2E tests.
|
||||
|
||||
Each subcommand is a self-contained check that prints a sentinel on success.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from cleveragents.acp.errors import ( # noqa: E402
|
||||
AcpNotAvailableError,
|
||||
AcpOperationNotFoundError,
|
||||
AcpVersionMismatchError,
|
||||
)
|
||||
from cleveragents.acp.events import AcpEventQueue # noqa: E402
|
||||
from cleveragents.acp.facade import AcpLocalFacade # noqa: E402
|
||||
from cleveragents.acp.models import AcpEvent, AcpRequest # noqa: E402
|
||||
from cleveragents.acp.transport import AcpHttpTransport # noqa: E402
|
||||
from cleveragents.acp.versioning import AcpVersionNegotiator # noqa: E402
|
||||
from cleveragents.application.services.automation_profile_service import ( # noqa: E402
|
||||
AutomationProfileService,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_guard import ( # noqa: E402
|
||||
AutomationGuard,
|
||||
)
|
||||
from cleveragents.domain.models.core.automation_profile import ( # noqa: E402
|
||||
AutomationProfile,
|
||||
)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m6"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def facade_session() -> None:
|
||||
"""Dispatch session.create and session.close."""
|
||||
facade = AcpLocalFacade()
|
||||
|
||||
resp_create = facade.dispatch(AcpRequest(operation="session.create", params={}))
|
||||
assert resp_create.status == "ok", f"Expected ok, got {resp_create.status}"
|
||||
assert "session_id" in resp_create.data
|
||||
|
||||
resp_close = facade.dispatch(AcpRequest(operation="session.close", params={}))
|
||||
assert resp_close.status == "ok"
|
||||
assert resp_close.data["status"] == "closed"
|
||||
|
||||
print("m6-facade-session-ok")
|
||||
|
||||
|
||||
def facade_plan() -> None:
|
||||
"""Dispatch plan create/execute/status/diff/apply."""
|
||||
facade = AcpLocalFacade()
|
||||
|
||||
resp = facade.dispatch(AcpRequest(operation="plan.create", params={}))
|
||||
assert resp.status == "ok"
|
||||
plan_id = resp.data["plan_id"]
|
||||
assert plan_id
|
||||
|
||||
resp = facade.dispatch(
|
||||
AcpRequest(operation="plan.execute", params={"plan_id": plan_id})
|
||||
)
|
||||
assert resp.status == "ok"
|
||||
assert resp.data["status"] == "queued"
|
||||
|
||||
resp = facade.dispatch(
|
||||
AcpRequest(operation="plan.status", params={"plan_id": plan_id})
|
||||
)
|
||||
assert resp.status == "ok"
|
||||
assert "phase" in resp.data
|
||||
|
||||
resp = facade.dispatch(
|
||||
AcpRequest(operation="plan.diff", params={"plan_id": plan_id})
|
||||
)
|
||||
assert resp.status == "ok"
|
||||
assert "changes" in resp.data
|
||||
|
||||
resp = facade.dispatch(
|
||||
AcpRequest(operation="plan.apply", params={"plan_id": plan_id})
|
||||
)
|
||||
assert resp.status == "ok"
|
||||
assert resp.data["status"] == "applied"
|
||||
|
||||
print("m6-facade-plan-ok")
|
||||
|
||||
|
||||
def facade_unknown_op() -> None:
|
||||
"""Verify unknown operation raises AcpOperationNotFoundError."""
|
||||
facade = AcpLocalFacade()
|
||||
try:
|
||||
facade.dispatch(AcpRequest(operation="nonexistent.op", params={}))
|
||||
print("FAIL: expected AcpOperationNotFoundError")
|
||||
sys.exit(1)
|
||||
except AcpOperationNotFoundError:
|
||||
pass
|
||||
|
||||
print("m6-facade-unknown-op-ok")
|
||||
|
||||
|
||||
def event_queue() -> None:
|
||||
"""Publish events and verify local subscriber receives them."""
|
||||
queue = AcpEventQueue()
|
||||
received = []
|
||||
|
||||
sub_id = queue.subscribe_local(lambda e: received.append(e))
|
||||
queue.publish(AcpEvent(event_type="plan.progress", data={"step": 1}))
|
||||
assert len(received) == 1
|
||||
assert received[0].event_type == "plan.progress"
|
||||
|
||||
# Unsubscribe and verify no more callbacks
|
||||
queue.unsubscribe(sub_id)
|
||||
queue.publish(AcpEvent(event_type="plan.complete", data={}))
|
||||
assert len(received) == 1 # still 1
|
||||
|
||||
# Verify get_events returns both
|
||||
events = queue.get_events()
|
||||
assert len(events) == 2
|
||||
|
||||
# Close and verify publish raises
|
||||
queue.close()
|
||||
try:
|
||||
queue.publish(AcpEvent(event_type="after.close", data={}))
|
||||
print("FAIL: expected RuntimeError")
|
||||
sys.exit(1)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
print("m6-event-queue-ok")
|
||||
|
||||
|
||||
def transport_stub() -> None:
|
||||
"""Verify HTTP transport stub raises AcpNotAvailableError."""
|
||||
transport = AcpHttpTransport()
|
||||
|
||||
# send
|
||||
try:
|
||||
transport.send(AcpRequest(operation="plan.create", params={}))
|
||||
print("FAIL: expected AcpNotAvailableError on send")
|
||||
sys.exit(1)
|
||||
except AcpNotAvailableError:
|
||||
pass
|
||||
|
||||
# connect
|
||||
try:
|
||||
transport.connect("https://example.com/acp")
|
||||
print("FAIL: expected AcpNotAvailableError on connect")
|
||||
sys.exit(1)
|
||||
except AcpNotAvailableError:
|
||||
pass
|
||||
|
||||
# disconnect
|
||||
try:
|
||||
transport.disconnect()
|
||||
print("FAIL: expected AcpNotAvailableError on disconnect")
|
||||
sys.exit(1)
|
||||
except AcpNotAvailableError:
|
||||
pass
|
||||
|
||||
# is_connected
|
||||
assert transport.is_connected() is False
|
||||
|
||||
print("m6-transport-stub-ok")
|
||||
|
||||
|
||||
def version_negotiation() -> None:
|
||||
"""Negotiate supported and unsupported ACP versions."""
|
||||
negotiator = AcpVersionNegotiator()
|
||||
|
||||
# Supported
|
||||
result = negotiator.negotiate("1.0")
|
||||
assert result == "1.0"
|
||||
assert negotiator.is_supported("1.0") is True
|
||||
assert negotiator.get_current() == "1.0"
|
||||
|
||||
# Unsupported
|
||||
try:
|
||||
negotiator.negotiate("2.0")
|
||||
print("FAIL: expected AcpVersionMismatchError")
|
||||
sys.exit(1)
|
||||
except AcpVersionMismatchError:
|
||||
pass
|
||||
|
||||
assert negotiator.is_supported("99.0") is False
|
||||
|
||||
print("m6-version-negotiation-ok")
|
||||
|
||||
|
||||
def guard_denylist() -> None:
|
||||
"""Verify denylist guard blocks denied tools."""
|
||||
guard = AutomationGuard(tool_denylist=["rm_rf", "drop_database"])
|
||||
profile = AutomationProfile(name="test-deny", guards=guard)
|
||||
|
||||
# Allowed tool
|
||||
result = profile.check_guard(tool_name="read_file")
|
||||
assert result.allowed is True
|
||||
|
||||
# Denied tool
|
||||
result = profile.check_guard(tool_name="rm_rf")
|
||||
assert result.allowed is False
|
||||
assert result.requires_approval is True
|
||||
assert "denylist" in (result.reason or "")
|
||||
|
||||
print("m6-guard-denylist-ok")
|
||||
|
||||
|
||||
def guard_budget() -> None:
|
||||
"""Verify cost budget and call limit guards."""
|
||||
guard = AutomationGuard(max_total_cost=10.0, max_tool_calls_per_step=5)
|
||||
profile = AutomationProfile(name="test-budget", guards=guard)
|
||||
|
||||
# Under budget and limit
|
||||
result = profile.check_guard(tool_name="llm_call", cost_so_far=3.0, calls_so_far=2)
|
||||
assert result.allowed is True
|
||||
|
||||
# Over budget
|
||||
result = profile.check_guard(tool_name="llm_call", cost_so_far=10.0, calls_so_far=2)
|
||||
assert result.allowed is False
|
||||
assert (
|
||||
"budget" in (result.reason or "").lower()
|
||||
or "cost" in (result.reason or "").lower()
|
||||
)
|
||||
|
||||
# Over call limit
|
||||
result = profile.check_guard(tool_name="llm_call", cost_so_far=3.0, calls_so_far=5)
|
||||
assert result.allowed is False
|
||||
assert "limit" in (result.reason or "").lower()
|
||||
|
||||
print("m6-guard-budget-ok")
|
||||
|
||||
|
||||
def profile_resolution() -> None:
|
||||
"""Verify plan > action > project > global resolution."""
|
||||
service = AutomationProfileService()
|
||||
|
||||
# Plan takes precedence
|
||||
p = service.resolve_profile(
|
||||
plan_profile="ci", action_profile="auto", project_profile="manual"
|
||||
)
|
||||
assert p.name == "ci"
|
||||
|
||||
# Action takes precedence over project
|
||||
p = service.resolve_profile(
|
||||
plan_profile=None, action_profile="auto", project_profile="manual"
|
||||
)
|
||||
assert p.name == "auto"
|
||||
|
||||
# Project takes precedence over global
|
||||
p = service.resolve_profile(
|
||||
plan_profile=None, action_profile=None, project_profile="review"
|
||||
)
|
||||
assert p.name == "review"
|
||||
|
||||
# Falls back to global default (manual)
|
||||
p = service.resolve_profile(
|
||||
plan_profile=None, action_profile=None, project_profile=None
|
||||
)
|
||||
assert p.name == "manual"
|
||||
|
||||
print("m6-profile-resolution-ok")
|
||||
|
||||
|
||||
def fixture_loading() -> None:
|
||||
"""Load all M6 fixture files and verify structure."""
|
||||
for fname in (
|
||||
"acp_facade_flows.json",
|
||||
"autonomy_guardrails.json",
|
||||
"automation_profiles.json",
|
||||
):
|
||||
fpath = _FIXTURES_DIR / fname
|
||||
with open(fpath) as f:
|
||||
data = json.load(f)
|
||||
assert "fixtures" in data, f"Missing 'fixtures' key in {fname}"
|
||||
assert len(data["fixtures"]) > 0, f"Empty fixtures in {fname}"
|
||||
print("m6-fixture-loading-ok")
|
||||
|
||||
|
||||
def full_flow() -> None:
|
||||
"""End-to-end: facade dispatch + guard check + profile resolution."""
|
||||
# Step 1: Facade dispatch
|
||||
facade = AcpLocalFacade()
|
||||
resp = facade.dispatch(AcpRequest(operation="session.create", params={}))
|
||||
assert resp.status == "ok"
|
||||
|
||||
resp = facade.dispatch(AcpRequest(operation="plan.create", params={}))
|
||||
assert resp.status == "ok"
|
||||
plan_id = resp.data["plan_id"]
|
||||
|
||||
resp = facade.dispatch(
|
||||
AcpRequest(operation="plan.execute", params={"plan_id": plan_id})
|
||||
)
|
||||
assert resp.status == "ok"
|
||||
|
||||
# Step 2: Guard check
|
||||
guard = AutomationGuard(
|
||||
tool_denylist=["dangerous_tool"],
|
||||
max_tool_calls_per_step=3,
|
||||
)
|
||||
profile = AutomationProfile(name="test-full-flow", guards=guard)
|
||||
|
||||
result = profile.check_guard(tool_name="safe_tool", calls_so_far=1)
|
||||
assert result.allowed is True
|
||||
|
||||
result = profile.check_guard(tool_name="dangerous_tool")
|
||||
assert result.allowed is False
|
||||
|
||||
# Step 3: Profile resolution
|
||||
service = AutomationProfileService()
|
||||
p = service.resolve_profile(plan_profile="ci")
|
||||
assert p.name == "ci"
|
||||
|
||||
# Step 4: Version negotiation
|
||||
negotiator = AcpVersionNegotiator()
|
||||
assert negotiator.negotiate("1.0") == "1.0"
|
||||
|
||||
# Step 5: Event queue
|
||||
queue = AcpEventQueue()
|
||||
queue.publish(AcpEvent(event_type="flow.complete", data={}))
|
||||
assert len(queue.get_events()) == 1
|
||||
queue.close()
|
||||
|
||||
print("m6-full-flow-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"facade-session": facade_session,
|
||||
"facade-plan": facade_plan,
|
||||
"facade-unknown-op": facade_unknown_op,
|
||||
"event-queue": event_queue,
|
||||
"transport-stub": transport_stub,
|
||||
"version-negotiation": version_negotiation,
|
||||
"guard-denylist": guard_denylist,
|
||||
"guard-budget": guard_budget,
|
||||
"profile-resolution": profile_resolution,
|
||||
"fixture-loading": fixture_loading,
|
||||
"full-flow": full_flow,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
|
||||
sys.exit(1)
|
||||
fn = _COMMANDS[sys.argv[1]]
|
||||
fn() # type: ignore[operator]
|
||||
@@ -0,0 +1,97 @@
|
||||
*** Settings ***
|
||||
Documentation M6 autonomy acceptance E2E smoke tests — ACP facade, guardrails, audit
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m6_autonomy_acceptance.py
|
||||
|
||||
*** Test Cases ***
|
||||
M6 ACP Facade Session Lifecycle
|
||||
[Documentation] Dispatch session.create and session.close via local facade
|
||||
${result}= Run Process ${PYTHON} ${HELPER} facade-session cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-facade-session-ok
|
||||
|
||||
M6 ACP Facade Plan Lifecycle
|
||||
[Documentation] Dispatch plan create/execute/status/diff/apply operations
|
||||
${result}= Run Process ${PYTHON} ${HELPER} facade-plan cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-facade-plan-ok
|
||||
|
||||
M6 ACP Facade Unknown Operation Error
|
||||
[Documentation] Verify unknown operations raise AcpOperationNotFoundError
|
||||
${result}= Run Process ${PYTHON} ${HELPER} facade-unknown-op cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-facade-unknown-op-ok
|
||||
|
||||
M6 ACP Event Queue Publish Subscribe
|
||||
[Documentation] Publish events and verify local subscriber receives them
|
||||
${result}= Run Process ${PYTHON} ${HELPER} event-queue cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-event-queue-ok
|
||||
|
||||
M6 ACP Transport Stub Rejects All
|
||||
[Documentation] Verify HTTP transport stub raises AcpNotAvailableError
|
||||
${result}= Run Process ${PYTHON} ${HELPER} transport-stub cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-transport-stub-ok
|
||||
|
||||
M6 ACP Version Negotiation
|
||||
[Documentation] Negotiate supported and unsupported ACP versions
|
||||
${result}= Run Process ${PYTHON} ${HELPER} version-negotiation cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-version-negotiation-ok
|
||||
|
||||
M6 Guard Denylist Enforcement
|
||||
[Documentation] Verify denylist guard blocks denied tools
|
||||
${result}= Run Process ${PYTHON} ${HELPER} guard-denylist cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-guard-denylist-ok
|
||||
|
||||
M6 Guard Budget Enforcement
|
||||
[Documentation] Verify cost budget and call limit guards work
|
||||
${result}= Run Process ${PYTHON} ${HELPER} guard-budget cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-guard-budget-ok
|
||||
|
||||
M6 Profile Resolution Precedence
|
||||
[Documentation] Verify plan > action > project > global resolution
|
||||
${result}= Run Process ${PYTHON} ${HELPER} profile-resolution cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-profile-resolution-ok
|
||||
|
||||
M6 Fixture Loading
|
||||
[Documentation] Load all M6 fixture files and verify structure
|
||||
${result}= Run Process ${PYTHON} ${HELPER} fixture-loading cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-fixture-loading-ok
|
||||
|
||||
M6 Full Autonomy Acceptance Flow
|
||||
[Documentation] End-to-end: facade dispatch + guard check + profile resolution
|
||||
${result}= Run Process ${PYTHON} ${HELPER} full-flow cwd=${WORKSPACE} timeout=60s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m6-full-flow-ok
|
||||
Reference in New Issue
Block a user