test(cli): cover action and plan extensions
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s
Added comprehensive test coverage for CLI extension features from #325: - Behave scenarios for automation_profile resolution, invariant ordering, and actor override error cases - Output snapshot assertions for JSON/YAML/table formats - Robot integration test for action show with optional actors/invariants - ASV benchmark for extended CLI scenario runtime baseline - Updated testing.md with CLI extension test fixture documentation ISSUES CLOSED: #326
This commit is contained in:
@@ -0,0 +1,324 @@
|
||||
"""ASV benchmarks for CLI extension test scenario runtime baselines.
|
||||
|
||||
Benchmarks the extended test scenarios added in #326:
|
||||
- Automation profile resolution across all builtin profiles
|
||||
- Invariant ordering verification with multiple flags
|
||||
- Actor override validation (valid and error paths)
|
||||
- Output format rendering (JSON, YAML, table) with extended fields
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import ( # noqa: E402
|
||||
validate_namespaced_actor,
|
||||
)
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
AutomationProfileProvenance,
|
||||
AutomationProfileRef,
|
||||
InvariantSource,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanInvariant,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_runner = CliRunner()
|
||||
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
||||
|
||||
_BUILTIN_PROFILE_NAMES = [
|
||||
"manual",
|
||||
"review",
|
||||
"supervised",
|
||||
"cautious",
|
||||
"trusted",
|
||||
"auto",
|
||||
]
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/bench-action") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark action",
|
||||
long_description=None,
|
||||
definition_of_done="Benchmarks pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
name: str = "local/bench-plan",
|
||||
*,
|
||||
with_profile: bool = False,
|
||||
with_invariants: bool = False,
|
||||
invariant_count: int = 2,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
profile = None
|
||||
if with_profile:
|
||||
profile = AutomationProfileRef(
|
||||
profile_name="trusted",
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
)
|
||||
invariants: list[PlanInvariant] = []
|
||||
if with_invariants:
|
||||
invariants = [
|
||||
PlanInvariant(
|
||||
text=f"Invariant {i}",
|
||||
source=InvariantSource.PLAN,
|
||||
)
|
||||
for i in range(invariant_count)
|
||||
]
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark plan",
|
||||
definition_of_done="Benchmarks pass",
|
||||
action_name="local/bench-action",
|
||||
phase=PlanPhase.STRATEGIZE,
|
||||
processing_state=ProcessingState.QUEUED,
|
||||
project_links=[ProjectLink(project_name="my-project")],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
automation_profile=profile,
|
||||
invariants=invariants,
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
class ProfileResolutionSuite:
|
||||
"""Benchmark automation profile resolution across all builtin profiles."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan(with_profile=True)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_resolve_all_builtin_profiles(self) -> None:
|
||||
"""Benchmark resolving each builtin profile in sequence."""
|
||||
for profile_name in _BUILTIN_PROFILE_NAMES:
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--automation-profile",
|
||||
profile_name,
|
||||
],
|
||||
)
|
||||
|
||||
def time_resolve_single_profile(self) -> None:
|
||||
"""Benchmark resolving a single profile."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--automation-profile", "trusted"],
|
||||
)
|
||||
|
||||
def time_reject_invalid_profile(self) -> None:
|
||||
"""Benchmark rejecting an invalid profile name."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--automation-profile", "nonexistent"],
|
||||
)
|
||||
|
||||
|
||||
class InvariantOrderingSuite:
|
||||
"""Benchmark invariant ordering verification with multiple flags."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan(
|
||||
with_invariants=True, invariant_count=5
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_three_invariants(self) -> None:
|
||||
"""Benchmark plan use with three invariant flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--invariant",
|
||||
"Alpha",
|
||||
"--invariant",
|
||||
"Beta",
|
||||
"--invariant",
|
||||
"Gamma",
|
||||
],
|
||||
)
|
||||
|
||||
def time_five_invariants(self) -> None:
|
||||
"""Benchmark plan use with five invariant flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--invariant",
|
||||
"A",
|
||||
"--invariant",
|
||||
"B",
|
||||
"--invariant",
|
||||
"C",
|
||||
"--invariant",
|
||||
"D",
|
||||
"--invariant",
|
||||
"E",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class ActorValidationErrorSuite:
|
||||
"""Benchmark actor override validation for error paths."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_valid_actor_validation(self) -> None:
|
||||
"""Benchmark validating a valid actor name."""
|
||||
validate_namespaced_actor("openai/gpt-4", "--strategy-actor")
|
||||
|
||||
def time_invalid_actor_rejection(self) -> None:
|
||||
"""Benchmark rejecting an invalid actor name."""
|
||||
try:
|
||||
validate_namespaced_actor("bad-format", "--strategy-actor")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def time_plan_use_invalid_actor_cli(self) -> None:
|
||||
"""Benchmark CLI rejection of invalid actor flag."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--strategy-actor", "bad-format"],
|
||||
)
|
||||
|
||||
|
||||
class OutputFormatRenderingSuite:
|
||||
"""Benchmark output rendering in JSON/YAML/table with extended fields."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
plan = _mock_plan(with_profile=True, with_invariants=True)
|
||||
self._mock_service.list_plans.return_value = [plan]
|
||||
self._mock_service.get_plan.return_value = plan
|
||||
self._plan_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._action_patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._plan_patcher.start()
|
||||
self._action_patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._plan_patcher.stop()
|
||||
self._action_patcher.stop()
|
||||
|
||||
def time_action_show_json(self) -> None:
|
||||
"""Benchmark action show in JSON format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "json"],
|
||||
)
|
||||
|
||||
def time_action_show_yaml(self) -> None:
|
||||
"""Benchmark action show in YAML format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "yaml"],
|
||||
)
|
||||
|
||||
def time_action_show_table(self) -> None:
|
||||
"""Benchmark action show in table format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "table"],
|
||||
)
|
||||
|
||||
def time_plan_status_json(self) -> None:
|
||||
"""Benchmark plan status in JSON format."""
|
||||
_runner.invoke(plan_app, ["status", "--format", "json"])
|
||||
|
||||
def time_plan_status_yaml(self) -> None:
|
||||
"""Benchmark plan status in YAML format."""
|
||||
_runner.invoke(plan_app, ["status", "--format", "yaml"])
|
||||
|
||||
def time_plan_lifecycle_list_json(self) -> None:
|
||||
"""Benchmark plan lifecycle-list in JSON format."""
|
||||
_runner.invoke(plan_app, ["lifecycle-list", "--format", "json"])
|
||||
@@ -55,17 +55,21 @@ def _mock_action(name: str = "local/bench-action") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark action",
|
||||
long_description=None,
|
||||
definition_of_done="Benchmarks pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
@@ -105,6 +109,9 @@ def _mock_plan(
|
||||
invariants=invariants,
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
@@ -714,3 +714,133 @@ nox -s benchmark
|
||||
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
|
||||
|
||||
For full scale testing documentation, see `docs/development/scale_testing.md`.
|
||||
|
||||
## CLI Extension Test Fixtures
|
||||
|
||||
The CLI extension tests cover automation profile, invariant, and actor override
|
||||
functionality added in #325. Deeper coverage added in #326 validates resolution
|
||||
logic, ordering semantics, error cases, and output format snapshots.
|
||||
|
||||
### Test Fixture Patterns
|
||||
|
||||
The CLI extension test suites use a **mocked lifecycle service** pattern. The
|
||||
`_get_lifecycle_service` function in both `plan.py` and `action.py` is patched
|
||||
so that tests exercise real CLI argument parsing, validation, and output
|
||||
rendering without requiring a database or running server.
|
||||
|
||||
#### Common Fixtures
|
||||
|
||||
| Fixture | Module | Description |
|
||||
|---------|--------|-------------|
|
||||
| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides |
|
||||
| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) |
|
||||
| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead |
|
||||
| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations |
|
||||
|
||||
#### Automation Profile Resolution Fixtures
|
||||
|
||||
Tests validate that each builtin profile name (`manual`, `review`, `supervised`,
|
||||
`cautious`, `trusted`, `auto`) resolves correctly through the CLI and that
|
||||
invalid profile names are rejected with a helpful error listing available
|
||||
profiles. The service mock returns a `Plan` with an `AutomationProfileRef` to
|
||||
verify end-to-end profile assignment.
|
||||
|
||||
#### Invariant Ordering Fixtures
|
||||
|
||||
Tests pass multiple `--invariant` flags and verify the service receives them in
|
||||
exact insertion order. This tests the list-accumulation behaviour of Typer's
|
||||
repeated option handling.
|
||||
|
||||
#### Actor Override Error Fixtures
|
||||
|
||||
Tests exercise the `validate_namespaced_actor` regex
|
||||
(`^[a-z][a-z0-9-]*/[a-z][a-z0-9._-]*$`) with various invalid inputs:
|
||||
- Empty string
|
||||
- Missing slash (`no-slash`)
|
||||
- Uppercase characters (`UPPER/case`)
|
||||
- Leading slash (`/leading-slash`)
|
||||
- Trailing slash (`trail/ing/`)
|
||||
- Double slashes (`open//ai`)
|
||||
- Special characters (`open@ai/gpt-4`)
|
||||
- Numeric-leading namespace (`123/model`)
|
||||
- Whitespace (`open ai/gpt`)
|
||||
|
||||
#### Output Snapshot Assertions
|
||||
|
||||
Tests render plan status, lifecycle-list, and action show output in JSON, YAML,
|
||||
and table formats, then assert that:
|
||||
- JSON output is valid (parses without error)
|
||||
- Required keys are present (`namespaced_name`, `phase`, `automation_profile`, etc.)
|
||||
- Invariant text values survive serialization round-trips
|
||||
- YAML output contains expected field names
|
||||
- Table output contains human-readable field values
|
||||
|
||||
### Behave Suite: `features/cli_extensions.feature`
|
||||
|
||||
Covers scenarios across these areas:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Automation profile flags | 2 | Valid and invalid profile names |
|
||||
| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty |
|
||||
| Invariant flags | 2 | Single and dual invariant flags |
|
||||
| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation |
|
||||
| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors |
|
||||
| Actor overrides (invalid) | 4 | Malformed namespace/name formats |
|
||||
| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace |
|
||||
| Plan status display | 4 | Table, JSON, single plan with invariants/profile |
|
||||
| Plan lifecycle-list | 2 | Table and JSON with invariant count |
|
||||
| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile |
|
||||
| Combined flags | 1 | Profile + invariant together |
|
||||
| validate_namespaced_actor | 2 | Unit-level accept/reject assertions |
|
||||
| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action |
|
||||
|
||||
Step definitions: `features/steps/cli_extensions_steps.py`
|
||||
|
||||
### Robot Suite: `robot/cli_extensions.robot`
|
||||
|
||||
Integration tests exercising CLI extensions through Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly |
|
||||
| Plan Use With Automation Profile | Verifies `--automation-profile` works |
|
||||
| Plan Use Actor Validation Valid | Valid namespaced actor format accepted |
|
||||
| Plan Use Actor Validation Invalid | Invalid actor format rejected |
|
||||
| Plan Use Combined Profile And Invariants | Both profile and invariants together |
|
||||
| Action Show With Optional Actors And Invariants | `action show` renders estimation_actor, invariant_actor, invariants, inputs_schema |
|
||||
| Action Show JSON With Optional Fields | JSON output includes all optional fields |
|
||||
| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order |
|
||||
| Profile Resolution All Builtins | All builtin profile names resolve correctly |
|
||||
|
||||
Helper script: `robot/helper_cli_extensions.py`
|
||||
|
||||
### ASV Benchmarks
|
||||
|
||||
Two benchmark files cover CLI extension performance:
|
||||
|
||||
**`benchmarks/cli_extensions_bench.py`** (original):
|
||||
- `PlanUseWithProfileSuite` — plan use with automation profile
|
||||
- `PlanUseWithInvariantsSuite` — plan use with invariant flags
|
||||
- `PlanUseActorValidationSuite` — plan use with actor overrides
|
||||
- `PlanStatusWithExtendedFieldsSuite` — status/lifecycle-list rendering
|
||||
- `ActionShowExtendedSuite` — action show in rich and JSON formats
|
||||
|
||||
**`benchmarks/cli_extension_tests_bench.py`** (extended #326):
|
||||
- `ProfileResolutionSuite` — resolve all builtin profiles, single profile, invalid profile
|
||||
- `InvariantOrderingSuite` — three and five invariant flag parsing
|
||||
- `ActorValidationErrorSuite` — valid/invalid actor regex, CLI rejection
|
||||
- `OutputFormatRenderingSuite` — JSON/YAML/table rendering for action show and plan status
|
||||
|
||||
### Running CLI Extension Tests
|
||||
|
||||
```bash
|
||||
# Behave only (CLI extensions feature)
|
||||
nox -s unit_tests -- features/cli_extensions.feature
|
||||
|
||||
# Robot only (CLI extensions suite)
|
||||
nox -s integration_tests -- --suite robot/cli_extensions.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
@@ -189,3 +189,175 @@ Feature: CLI extensions for plan and action commands
|
||||
And validate_namespaced_actor rejects "UPPER/case"
|
||||
And validate_namespaced_actor rejects "/leading-slash"
|
||||
And validate_namespaced_actor rejects "trail/ing/"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Automation profile resolution logic (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "manual"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "manual"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "manual"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "review"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "review"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "review"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "supervised"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "supervised"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "supervised"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "cautious"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "cautious"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "cautious"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "auto"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "auto"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "auto"
|
||||
|
||||
Scenario: Plan use rejects profile name with special characters
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "tr@sted!"
|
||||
Then the cli extensions plan use should fail
|
||||
|
||||
Scenario: Plan use rejects profile name with spaces
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "my profile"
|
||||
Then the cli extensions plan use should fail
|
||||
|
||||
Scenario: Plan use profile resolution provides available profiles on error
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "nonexistent"
|
||||
Then the cli extensions plan use should fail
|
||||
And the cli extensions error output mentions available profiles
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Invariant ordering preservation (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use with three invariants preserves insertion order
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with three invariants "Alpha first" and "Beta second" and "Gamma third"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions service received invariants in order "Alpha first" then "Beta second" then "Gamma third"
|
||||
|
||||
Scenario: Plan use with five invariants preserves insertion order
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with five invariants "A" and "B" and "C" and "D" and "E"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions service received five invariants in exact order "A" "B" "C" "D" "E"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Actor override error cases (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use rejects actor with double slashes
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "open//ai"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with special characters in namespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "open@ai/gpt-4"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with numeric-start namespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "123/model"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with only a slash
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "/"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with whitespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with execution actor flag "open ai/gpt"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with missing name after slash
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with estimation actor flag "openai/"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: validate_namespaced_actor rejects double slash
|
||||
Then validate_namespaced_actor rejects "ns//name"
|
||||
|
||||
Scenario: validate_namespaced_actor rejects special characters
|
||||
Then validate_namespaced_actor rejects "ns@special/name"
|
||||
And validate_namespaced_actor rejects "ns/name#bad"
|
||||
|
||||
Scenario: validate_namespaced_actor rejects numeric-leading namespace
|
||||
Then validate_namespaced_actor rejects "1abc/model"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Output snapshot assertions JSON/YAML/table (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan status in JSON format contains required keys
|
||||
Given cli extensions plans exist with automation profile "trusted"
|
||||
When I run cli extensions plan status with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "automation_profile"
|
||||
And the cli extensions json output contains key "namespaced_name"
|
||||
And the cli extensions json output contains key "phase"
|
||||
|
||||
Scenario: Plan status in YAML format contains required fields
|
||||
Given cli extensions plans exist with automation profile "trusted"
|
||||
When I run cli extensions plan status with format "yaml"
|
||||
Then the cli extensions output contains yaml field "automation_profile"
|
||||
And the cli extensions output contains yaml field "namespaced_name"
|
||||
|
||||
Scenario: Action show in JSON format contains all required keys
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "estimation_actor"
|
||||
And the cli extensions json output contains key "strategy_actor"
|
||||
And the cli extensions json output contains key "execution_actor"
|
||||
And the cli extensions json output contains key "description"
|
||||
|
||||
Scenario: Action show in YAML format contains required fields
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "yaml"
|
||||
Then the cli extensions output contains yaml field "estimation_actor"
|
||||
And the cli extensions output contains yaml field "strategy_actor"
|
||||
|
||||
Scenario: Action show in JSON includes invariants list when present
|
||||
Given a cli extensions action with invariants
|
||||
When I run cli extensions action show with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "invariants"
|
||||
And the cli extensions json invariants list has length 2
|
||||
|
||||
Scenario: Plan lifecycle-list in YAML shows invariant data
|
||||
Given cli extensions plans exist with invariants
|
||||
When I run cli extensions plan lifecycle-list with format "yaml"
|
||||
Then the cli extensions output contains yaml field "invariants"
|
||||
|
||||
Scenario: Plan lifecycle-list in JSON preserves invariant text
|
||||
Given cli extensions plans exist with invariants
|
||||
When I run cli extensions plan lifecycle-list with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output string contains "No warnings"
|
||||
And the cli extensions json output string contains "Keep compat"
|
||||
|
||||
Scenario: Action show in table format contains actor information
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "table"
|
||||
Then the cli extensions action output should contain "openai/gpt-4"
|
||||
|
||||
Scenario: Plan status in table format contains profile name
|
||||
Given cli extensions plans exist with automation profile "cautious"
|
||||
When I run cli extensions plan status
|
||||
Then the cli extensions status output should contain "cautious"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -621,6 +622,188 @@ def step_validate_actor_rejects(context: Context, actor: str) -> None:
|
||||
pass # Expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Automation profile resolution (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the cli extensions error output mentions available profiles")
|
||||
def step_error_mentions_available(context: Context) -> None:
|
||||
"""Verify error output lists available automation profiles."""
|
||||
output = context.last_result.output
|
||||
# The error message should mention at least some builtin profiles
|
||||
assert "Available" in output or "manual" in output or "trusted" in output, (
|
||||
f"Expected available profiles in error output: {output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Invariant ordering (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I run plan use with three invariants "{inv1}" and "{inv2}" and "{inv3}"')
|
||||
def step_run_plan_use_three_invariants(
|
||||
context: Context, inv1: str, inv2: str, inv3: str
|
||||
) -> None:
|
||||
"""Run plan use with three --invariant flags."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
inv1,
|
||||
"--invariant",
|
||||
inv2,
|
||||
"--invariant",
|
||||
inv3,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then(
|
||||
"the cli extensions service received invariants in order "
|
||||
'"{inv1}" then "{inv2}" then "{inv3}"'
|
||||
)
|
||||
def step_service_received_ordered_invariants(
|
||||
context: Context, inv1: str, inv2: str, inv3: str
|
||||
) -> None:
|
||||
"""Verify invariants were passed in exact insertion order."""
|
||||
call_args = context.mock_service.use_action.call_args
|
||||
assert call_args is not None, "use_action was not called"
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
assert invariants is not None, "No invariants passed to use_action"
|
||||
texts = [inv.text for inv in invariants]
|
||||
assert len(texts) >= 3, f"Expected at least 3 invariants, got {len(texts)}"
|
||||
assert texts[0] == inv1, f"Expected '{inv1}' at index 0, got '{texts[0]}'"
|
||||
assert texts[1] == inv2, f"Expected '{inv2}' at index 1, got '{texts[1]}'"
|
||||
assert texts[2] == inv3, f"Expected '{inv3}' at index 2, got '{texts[2]}'"
|
||||
|
||||
|
||||
@when(
|
||||
"I run plan use with five invariants "
|
||||
'"{inv1}" and "{inv2}" and "{inv3}" and "{inv4}" and "{inv5}"'
|
||||
)
|
||||
def step_run_plan_use_five_invariants(
|
||||
context: Context,
|
||||
inv1: str,
|
||||
inv2: str,
|
||||
inv3: str,
|
||||
inv4: str,
|
||||
inv5: str,
|
||||
) -> None:
|
||||
"""Run plan use with five --invariant flags."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
inv1,
|
||||
"--invariant",
|
||||
inv2,
|
||||
"--invariant",
|
||||
inv3,
|
||||
"--invariant",
|
||||
inv4,
|
||||
"--invariant",
|
||||
inv5,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then(
|
||||
"the cli extensions service received five invariants in exact order "
|
||||
'"{inv1}" "{inv2}" "{inv3}" "{inv4}" "{inv5}"'
|
||||
)
|
||||
def step_service_received_five_ordered(
|
||||
context: Context,
|
||||
inv1: str,
|
||||
inv2: str,
|
||||
inv3: str,
|
||||
inv4: str,
|
||||
inv5: str,
|
||||
) -> None:
|
||||
"""Verify five invariants in exact insertion order."""
|
||||
call_args = context.mock_service.use_action.call_args
|
||||
assert call_args is not None, "use_action was not called"
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
assert invariants is not None, "No invariants passed"
|
||||
texts = [inv.text for inv in invariants]
|
||||
expected = [inv1, inv2, inv3, inv4, inv5]
|
||||
assert texts == expected, f"Expected {expected}, got {texts}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Output snapshot assertions (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the cli extensions output is valid JSON")
|
||||
def step_output_is_valid_json(context: Context) -> None:
|
||||
"""Verify last result output is valid JSON."""
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Exit code: {context.last_result.exit_code}, "
|
||||
f"output: {context.last_result.output}"
|
||||
)
|
||||
output = context.last_result.output.strip()
|
||||
try:
|
||||
context.parsed_json = json.loads(output)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(
|
||||
f"Output is not valid JSON: {exc}\nOutput: {output}"
|
||||
) from exc
|
||||
|
||||
|
||||
@then('the cli extensions json output contains key "{key}"')
|
||||
def step_json_output_contains_key(context: Context, key: str) -> None:
|
||||
"""Verify parsed JSON contains a specific key."""
|
||||
parsed = getattr(context, "parsed_json", None)
|
||||
if parsed is None:
|
||||
# Parse if not already parsed
|
||||
output = context.last_result.output.strip()
|
||||
parsed = json.loads(output)
|
||||
# Handle both dict and list-of-dict cases
|
||||
if isinstance(parsed, list):
|
||||
assert any(key in item for item in parsed if isinstance(item, dict)), (
|
||||
f"Key '{key}' not found in any list item"
|
||||
)
|
||||
else:
|
||||
assert key in parsed, f"Key '{key}' not in JSON: {list(parsed.keys())}"
|
||||
|
||||
|
||||
@then('the cli extensions output contains yaml field "{field}"')
|
||||
def step_output_contains_yaml_field(context: Context, field: str) -> None:
|
||||
"""Verify output contains a YAML field (key: value pattern)."""
|
||||
assert context.last_result.exit_code == 0
|
||||
output = context.last_result.output
|
||||
assert field in output, f"Expected YAML field '{field}' in output: {output}"
|
||||
|
||||
|
||||
@then("the cli extensions json invariants list has length {length:d}")
|
||||
def step_json_invariants_length(context: Context, length: int) -> None:
|
||||
"""Verify JSON invariants list has expected length."""
|
||||
parsed = getattr(context, "parsed_json", None)
|
||||
if parsed is None:
|
||||
output = context.last_result.output.strip()
|
||||
parsed = json.loads(output)
|
||||
invariants = parsed.get("invariants", [])
|
||||
assert len(invariants) == length, (
|
||||
f"Expected {length} invariants, got {len(invariants)}: {invariants}"
|
||||
)
|
||||
|
||||
|
||||
@then('the cli extensions json output string contains "{text}"')
|
||||
def step_json_output_string_contains(context: Context, text: str) -> None:
|
||||
"""Verify the raw JSON output string contains the given text."""
|
||||
assert context.last_result.exit_code == 0
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in JSON output: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -47,3 +47,35 @@ Plan Use Combined Profile And Invariants
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-plan-combined-ok
|
||||
|
||||
Action Show With Optional Actors And Invariants
|
||||
[Documentation] Verify action show includes optional actors, invariants, and inputs_schema when set
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-show-extended cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-action-show-extended-ok
|
||||
|
||||
Action Show JSON With Optional Fields
|
||||
[Documentation] Verify action show JSON output includes optional fields
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-show-json cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-action-show-json-ok
|
||||
|
||||
Invariant Ordering Preserved
|
||||
[Documentation] Verify that multiple invariant flags preserve insertion order
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-ordering cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-invariant-ordering-ok
|
||||
|
||||
Profile Resolution All Builtins
|
||||
[Documentation] Verify that all builtin profiles resolve correctly
|
||||
${result}= Run Process ${PYTHON} ${HELPER} profile-resolution cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-profile-resolution-ok
|
||||
|
||||
@@ -17,6 +17,7 @@ if _SRC not in sys.path:
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
@@ -62,6 +63,9 @@ def _mock_plan(
|
||||
invariants=invariants or [],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
@@ -70,10 +74,14 @@ def _mock_action() -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
@@ -228,6 +236,170 @@ def plan_use_combined() -> None:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _mock_action_extended() -> Action:
|
||||
"""Create an action with all optional fields set for extended show tests."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot extended",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
automation_profile="trusted",
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def action_show_extended() -> None:
|
||||
"""Verify action show renders optional actors, invariants, inputs_schema."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["show", "local/test-action"])
|
||||
output = result.output
|
||||
checks = [
|
||||
("Estimation Actor" in output or "estimation_actor" in output),
|
||||
("Invariant Actor" in output or "invariant_actor" in output),
|
||||
("Invariants" in output or "invariants" in output),
|
||||
("Inputs Schema" in output or "inputs_schema" in output),
|
||||
]
|
||||
if result.exit_code == 0 and all(checks):
|
||||
print("cli-ext-action-show-extended-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={output}")
|
||||
print(f"checks={checks}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def action_show_json() -> None:
|
||||
"""Verify action show JSON output includes optional fields."""
|
||||
import json
|
||||
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
action_app, ["show", "local/test-action", "--format", "json"]
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
try:
|
||||
data = json.loads(result.output.strip())
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}")
|
||||
sys.exit(1)
|
||||
required_keys = [
|
||||
"estimation_actor",
|
||||
"invariant_actor",
|
||||
"inputs_schema",
|
||||
"invariants",
|
||||
]
|
||||
missing = [k for k in required_keys if k not in data]
|
||||
if missing:
|
||||
print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-action-show-json-ok")
|
||||
|
||||
|
||||
def invariant_ordering() -> None:
|
||||
"""Verify that multiple invariant flags preserve insertion order."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
invariants=[
|
||||
PlanInvariant(text="Alpha", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Beta", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Gamma", source=InvariantSource.PLAN),
|
||||
]
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
"Alpha",
|
||||
"--invariant",
|
||||
"Beta",
|
||||
"--invariant",
|
||||
"Gamma",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
# Verify the service received invariants in order
|
||||
call_args = mock_svc.use_action.call_args
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
if invariants is None:
|
||||
print("FAIL: no invariants passed to use_action")
|
||||
sys.exit(1)
|
||||
texts = [inv.text for inv in invariants]
|
||||
if texts != ["Alpha", "Beta", "Gamma"]:
|
||||
print(f"FAIL: ordering mismatch: {texts}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-invariant-ordering-ok")
|
||||
|
||||
|
||||
def profile_resolution() -> None:
|
||||
"""Verify all builtin profiles resolve correctly."""
|
||||
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
|
||||
|
||||
for profile_name in sorted(BUILTIN_PROFILES):
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
automation_profile=AutomationProfileRef(
|
||||
profile_name=profile_name,
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--automation-profile",
|
||||
profile_name,
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(
|
||||
f"FAIL: profile '{profile_name}' "
|
||||
f"exit={result.exit_code} "
|
||||
f"output={result.output}"
|
||||
)
|
||||
sys.exit(1)
|
||||
print("cli-ext-profile-resolution-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -238,6 +410,10 @@ _COMMANDS: dict[str, object] = {
|
||||
"actor-valid": plan_use_actor_validation_valid,
|
||||
"actor-invalid": plan_use_actor_validation_invalid,
|
||||
"plan-combined": plan_use_combined,
|
||||
"action-show-extended": action_show_extended,
|
||||
"action-show-json": action_show_json,
|
||||
"invariant-ordering": invariant_ordering,
|
||||
"profile-resolution": profile_resolution,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user