test(cli): cover action and plan extensions
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s

Added comprehensive test coverage for CLI extension features from #325:
- Behave scenarios for automation_profile resolution, invariant ordering,
  and actor override error cases
- Output snapshot assertions for JSON/YAML/table formats
- Robot integration test for action show with optional actors/invariants
- ASV benchmark for extended CLI scenario runtime baseline
- Updated testing.md with CLI extension test fixture documentation

ISSUES CLOSED: #326
This commit is contained in:
2026-02-24 20:11:42 +00:00
parent c47e6445d0
commit 2c21611ad5
7 changed files with 1024 additions and 0 deletions
+324
View File
@@ -0,0 +1,324 @@
"""ASV benchmarks for CLI extension test scenario runtime baselines.
Benchmarks the extended test scenarios added in #326:
- Automation profile resolution across all builtin profiles
- Invariant ordering verification with multiple flags
- Actor override validation (valid and error paths)
- Output format rendering (JSON, YAML, table) with extended fields
"""
from __future__ import annotations
import importlib
import sys
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock, patch
# Ensure the local *source* tree is importable even when ASV has an
# older build of the package installed.
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.action import app as action_app # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.cli.commands.plan import ( # noqa: E402
validate_namespaced_actor,
)
from cleveragents.domain.models.core.action import ( # noqa: E402
Action,
ActionState,
)
from cleveragents.domain.models.core.plan import ( # noqa: E402
AutomationProfileProvenance,
AutomationProfileRef,
InvariantSource,
NamespacedName,
Plan,
PlanIdentity,
PlanInvariant,
PlanPhase,
PlanTimestamps,
ProcessingState,
ProjectLink,
)
_runner = CliRunner()
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
_BUILTIN_PROFILE_NAMES = [
"manual",
"review",
"supervised",
"cautious",
"trusted",
"auto",
]
def _mock_action(name: str = "local/bench-action") -> Action:
return Action(
namespaced_name=NamespacedName.parse(name),
description="Benchmark action",
long_description=None,
definition_of_done="Benchmarks pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor="openai/gpt-4",
invariant_actor="anthropic/claude-3",
reusable=True,
read_only=False,
invariants=["No regressions", "Keep backward compat"],
inputs_schema={
"type": "object",
"properties": {"coverage": {"type": "integer"}},
},
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
def _mock_plan(
name: str = "local/bench-plan",
*,
with_profile: bool = False,
with_invariants: bool = False,
invariant_count: int = 2,
) -> Plan:
now = datetime.now()
profile = None
if with_profile:
profile = AutomationProfileRef(
profile_name="trusted",
provenance=AutomationProfileProvenance.PLAN,
)
invariants: list[PlanInvariant] = []
if with_invariants:
invariants = [
PlanInvariant(
text=f"Invariant {i}",
source=InvariantSource.PLAN,
)
for i in range(invariant_count)
]
return Plan(
identity=PlanIdentity(plan_id=_PLAN_ULID),
namespaced_name=NamespacedName.parse(name),
description="Benchmark plan",
definition_of_done="Benchmarks pass",
action_name="local/bench-action",
phase=PlanPhase.STRATEGIZE,
processing_state=ProcessingState.QUEUED,
project_links=[ProjectLink(project_name="my-project")],
arguments={},
arguments_order=[],
automation_profile=profile,
invariants=invariants,
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
created_by=None,
timestamps=PlanTimestamps(created_at=now, updated_at=now),
)
class ProfileResolutionSuite:
"""Benchmark automation profile resolution across all builtin profiles."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.get_action_by_name.return_value = _mock_action()
self._mock_service.use_action.return_value = _mock_plan(with_profile=True)
self._patcher = patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=self._mock_service,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_resolve_all_builtin_profiles(self) -> None:
"""Benchmark resolving each builtin profile in sequence."""
for profile_name in _BUILTIN_PROFILE_NAMES:
_runner.invoke(
plan_app,
[
"use",
"local/bench-action",
"--automation-profile",
profile_name,
],
)
def time_resolve_single_profile(self) -> None:
"""Benchmark resolving a single profile."""
_runner.invoke(
plan_app,
["use", "local/bench-action", "--automation-profile", "trusted"],
)
def time_reject_invalid_profile(self) -> None:
"""Benchmark rejecting an invalid profile name."""
_runner.invoke(
plan_app,
["use", "local/bench-action", "--automation-profile", "nonexistent"],
)
class InvariantOrderingSuite:
"""Benchmark invariant ordering verification with multiple flags."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.get_action_by_name.return_value = _mock_action()
self._mock_service.use_action.return_value = _mock_plan(
with_invariants=True, invariant_count=5
)
self._patcher = patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=self._mock_service,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_three_invariants(self) -> None:
"""Benchmark plan use with three invariant flags."""
_runner.invoke(
plan_app,
[
"use",
"local/bench-action",
"--invariant",
"Alpha",
"--invariant",
"Beta",
"--invariant",
"Gamma",
],
)
def time_five_invariants(self) -> None:
"""Benchmark plan use with five invariant flags."""
_runner.invoke(
plan_app,
[
"use",
"local/bench-action",
"--invariant",
"A",
"--invariant",
"B",
"--invariant",
"C",
"--invariant",
"D",
"--invariant",
"E",
],
)
class ActorValidationErrorSuite:
"""Benchmark actor override validation for error paths."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.get_action_by_name.return_value = _mock_action()
self._mock_service.use_action.return_value = _mock_plan()
self._patcher = patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=self._mock_service,
)
self._patcher.start()
def teardown(self) -> None:
self._patcher.stop()
def time_valid_actor_validation(self) -> None:
"""Benchmark validating a valid actor name."""
validate_namespaced_actor("openai/gpt-4", "--strategy-actor")
def time_invalid_actor_rejection(self) -> None:
"""Benchmark rejecting an invalid actor name."""
try:
validate_namespaced_actor("bad-format", "--strategy-actor")
except Exception:
pass
def time_plan_use_invalid_actor_cli(self) -> None:
"""Benchmark CLI rejection of invalid actor flag."""
_runner.invoke(
plan_app,
["use", "local/bench-action", "--strategy-actor", "bad-format"],
)
class OutputFormatRenderingSuite:
"""Benchmark output rendering in JSON/YAML/table with extended fields."""
def setup(self) -> None:
self._mock_service = MagicMock()
self._mock_service.get_action_by_name.return_value = _mock_action()
plan = _mock_plan(with_profile=True, with_invariants=True)
self._mock_service.list_plans.return_value = [plan]
self._mock_service.get_plan.return_value = plan
self._plan_patcher = patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=self._mock_service,
)
self._action_patcher = patch(
"cleveragents.cli.commands.action._get_lifecycle_service",
return_value=self._mock_service,
)
self._plan_patcher.start()
self._action_patcher.start()
def teardown(self) -> None:
self._plan_patcher.stop()
self._action_patcher.stop()
def time_action_show_json(self) -> None:
"""Benchmark action show in JSON format."""
_runner.invoke(
action_app,
["show", "local/bench-action", "--format", "json"],
)
def time_action_show_yaml(self) -> None:
"""Benchmark action show in YAML format."""
_runner.invoke(
action_app,
["show", "local/bench-action", "--format", "yaml"],
)
def time_action_show_table(self) -> None:
"""Benchmark action show in table format."""
_runner.invoke(
action_app,
["show", "local/bench-action", "--format", "table"],
)
def time_plan_status_json(self) -> None:
"""Benchmark plan status in JSON format."""
_runner.invoke(plan_app, ["status", "--format", "json"])
def time_plan_status_yaml(self) -> None:
"""Benchmark plan status in YAML format."""
_runner.invoke(plan_app, ["status", "--format", "yaml"])
def time_plan_lifecycle_list_json(self) -> None:
"""Benchmark plan lifecycle-list in JSON format."""
_runner.invoke(plan_app, ["lifecycle-list", "--format", "json"])
+7
View File
@@ -55,17 +55,21 @@ def _mock_action(name: str = "local/bench-action") -> Action:
return Action(
namespaced_name=NamespacedName.parse(name),
description="Benchmark action",
long_description=None,
definition_of_done="Benchmarks pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor="openai/gpt-4",
invariant_actor="anthropic/claude-3",
reusable=True,
read_only=False,
invariants=["No regressions", "Keep backward compat"],
inputs_schema={
"type": "object",
"properties": {"coverage": {"type": "integer"}},
},
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
@@ -105,6 +109,9 @@ def _mock_plan(
invariants=invariants,
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
created_by=None,
timestamps=PlanTimestamps(created_at=now, updated_at=now),
)
+130
View File
@@ -714,3 +714,133 @@ nox -s benchmark
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
For full scale testing documentation, see `docs/development/scale_testing.md`.
## CLI Extension Test Fixtures
The CLI extension tests cover automation profile, invariant, and actor override
functionality added in #325. Deeper coverage added in #326 validates resolution
logic, ordering semantics, error cases, and output format snapshots.
### Test Fixture Patterns
The CLI extension test suites use a **mocked lifecycle service** pattern. The
`_get_lifecycle_service` function in both `plan.py` and `action.py` is patched
so that tests exercise real CLI argument parsing, validation, and output
rendering without requiring a database or running server.
#### Common Fixtures
| Fixture | Module | Description |
|---------|--------|-------------|
| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides |
| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) |
| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead |
| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations |
#### Automation Profile Resolution Fixtures
Tests validate that each builtin profile name (`manual`, `review`, `supervised`,
`cautious`, `trusted`, `auto`) resolves correctly through the CLI and that
invalid profile names are rejected with a helpful error listing available
profiles. The service mock returns a `Plan` with an `AutomationProfileRef` to
verify end-to-end profile assignment.
#### Invariant Ordering Fixtures
Tests pass multiple `--invariant` flags and verify the service receives them in
exact insertion order. This tests the list-accumulation behaviour of Typer's
repeated option handling.
#### Actor Override Error Fixtures
Tests exercise the `validate_namespaced_actor` regex
(`^[a-z][a-z0-9-]*/[a-z][a-z0-9._-]*$`) with various invalid inputs:
- Empty string
- Missing slash (`no-slash`)
- Uppercase characters (`UPPER/case`)
- Leading slash (`/leading-slash`)
- Trailing slash (`trail/ing/`)
- Double slashes (`open//ai`)
- Special characters (`open@ai/gpt-4`)
- Numeric-leading namespace (`123/model`)
- Whitespace (`open ai/gpt`)
#### Output Snapshot Assertions
Tests render plan status, lifecycle-list, and action show output in JSON, YAML,
and table formats, then assert that:
- JSON output is valid (parses without error)
- Required keys are present (`namespaced_name`, `phase`, `automation_profile`, etc.)
- Invariant text values survive serialization round-trips
- YAML output contains expected field names
- Table output contains human-readable field values
### Behave Suite: `features/cli_extensions.feature`
Covers scenarios across these areas:
| Area | Scenarios | Description |
|------|-----------|-------------|
| Automation profile flags | 2 | Valid and invalid profile names |
| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty |
| Invariant flags | 2 | Single and dual invariant flags |
| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation |
| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors |
| Actor overrides (invalid) | 4 | Malformed namespace/name formats |
| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace |
| Plan status display | 4 | Table, JSON, single plan with invariants/profile |
| Plan lifecycle-list | 2 | Table and JSON with invariant count |
| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile |
| Combined flags | 1 | Profile + invariant together |
| validate_namespaced_actor | 2 | Unit-level accept/reject assertions |
| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action |
Step definitions: `features/steps/cli_extensions_steps.py`
### Robot Suite: `robot/cli_extensions.robot`
Integration tests exercising CLI extensions through Python helper scripts:
| Test Case | Description |
|-----------|-------------|
| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly |
| Plan Use With Automation Profile | Verifies `--automation-profile` works |
| Plan Use Actor Validation Valid | Valid namespaced actor format accepted |
| Plan Use Actor Validation Invalid | Invalid actor format rejected |
| Plan Use Combined Profile And Invariants | Both profile and invariants together |
| Action Show With Optional Actors And Invariants | `action show` renders estimation_actor, invariant_actor, invariants, inputs_schema |
| Action Show JSON With Optional Fields | JSON output includes all optional fields |
| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order |
| Profile Resolution All Builtins | All builtin profile names resolve correctly |
Helper script: `robot/helper_cli_extensions.py`
### ASV Benchmarks
Two benchmark files cover CLI extension performance:
**`benchmarks/cli_extensions_bench.py`** (original):
- `PlanUseWithProfileSuite` — plan use with automation profile
- `PlanUseWithInvariantsSuite` — plan use with invariant flags
- `PlanUseActorValidationSuite` — plan use with actor overrides
- `PlanStatusWithExtendedFieldsSuite` — status/lifecycle-list rendering
- `ActionShowExtendedSuite` — action show in rich and JSON formats
**`benchmarks/cli_extension_tests_bench.py`** (extended #326):
- `ProfileResolutionSuite` — resolve all builtin profiles, single profile, invalid profile
- `InvariantOrderingSuite` — three and five invariant flag parsing
- `ActorValidationErrorSuite` — valid/invalid actor regex, CLI rejection
- `OutputFormatRenderingSuite` — JSON/YAML/table rendering for action show and plan status
### Running CLI Extension Tests
```bash
# Behave only (CLI extensions feature)
nox -s unit_tests -- features/cli_extensions.feature
# Robot only (CLI extensions suite)
nox -s integration_tests -- --suite robot/cli_extensions.robot
# Benchmarks
nox -s benchmark
```
+172
View File
@@ -189,3 +189,175 @@ Feature: CLI extensions for plan and action commands
And validate_namespaced_actor rejects "UPPER/case"
And validate_namespaced_actor rejects "/leading-slash"
And validate_namespaced_actor rejects "trail/ing/"
# ==========================================================================
# DEEP COVERAGE: Automation profile resolution logic (#326)
# ==========================================================================
Scenario: Plan use resolves valid builtin profile "manual"
Given a cli extensions action exists
When I run plan use with automation profile flag "manual"
Then the cli extensions plan use should succeed
And the cli extensions plan should have automation profile "manual"
Scenario: Plan use resolves valid builtin profile "review"
Given a cli extensions action exists
When I run plan use with automation profile flag "review"
Then the cli extensions plan use should succeed
And the cli extensions plan should have automation profile "review"
Scenario: Plan use resolves valid builtin profile "supervised"
Given a cli extensions action exists
When I run plan use with automation profile flag "supervised"
Then the cli extensions plan use should succeed
And the cli extensions plan should have automation profile "supervised"
Scenario: Plan use resolves valid builtin profile "cautious"
Given a cli extensions action exists
When I run plan use with automation profile flag "cautious"
Then the cli extensions plan use should succeed
And the cli extensions plan should have automation profile "cautious"
Scenario: Plan use resolves valid builtin profile "auto"
Given a cli extensions action exists
When I run plan use with automation profile flag "auto"
Then the cli extensions plan use should succeed
And the cli extensions plan should have automation profile "auto"
Scenario: Plan use rejects profile name with special characters
Given a cli extensions action exists
When I run plan use with automation profile flag "tr@sted!"
Then the cli extensions plan use should fail
Scenario: Plan use rejects profile name with spaces
Given a cli extensions action exists
When I run plan use with automation profile flag "my profile"
Then the cli extensions plan use should fail
Scenario: Plan use profile resolution provides available profiles on error
Given a cli extensions action exists
When I run plan use with automation profile flag "nonexistent"
Then the cli extensions plan use should fail
And the cli extensions error output mentions available profiles
# ==========================================================================
# DEEP COVERAGE: Invariant ordering preservation (#326)
# ==========================================================================
Scenario: Plan use with three invariants preserves insertion order
Given a cli extensions action exists
When I run plan use with three invariants "Alpha first" and "Beta second" and "Gamma third"
Then the cli extensions plan use should succeed
And the cli extensions service received invariants in order "Alpha first" then "Beta second" then "Gamma third"
Scenario: Plan use with five invariants preserves insertion order
Given a cli extensions action exists
When I run plan use with five invariants "A" and "B" and "C" and "D" and "E"
Then the cli extensions plan use should succeed
And the cli extensions service received five invariants in exact order "A" "B" "C" "D" "E"
# ==========================================================================
# DEEP COVERAGE: Actor override error cases (#326)
# ==========================================================================
Scenario: Plan use rejects actor with double slashes
Given a cli extensions action exists
When I run plan use with strategy actor flag "open//ai"
Then the cli extensions plan use should fail with actor validation error
Scenario: Plan use rejects actor with special characters in namespace
Given a cli extensions action exists
When I run plan use with strategy actor flag "open@ai/gpt-4"
Then the cli extensions plan use should fail with actor validation error
Scenario: Plan use rejects actor with numeric-start namespace
Given a cli extensions action exists
When I run plan use with strategy actor flag "123/model"
Then the cli extensions plan use should fail with actor validation error
Scenario: Plan use rejects actor with only a slash
Given a cli extensions action exists
When I run plan use with strategy actor flag "/"
Then the cli extensions plan use should fail with actor validation error
Scenario: Plan use rejects actor with whitespace
Given a cli extensions action exists
When I run plan use with execution actor flag "open ai/gpt"
Then the cli extensions plan use should fail with actor validation error
Scenario: Plan use rejects actor with missing name after slash
Given a cli extensions action exists
When I run plan use with estimation actor flag "openai/"
Then the cli extensions plan use should fail with actor validation error
Scenario: validate_namespaced_actor rejects double slash
Then validate_namespaced_actor rejects "ns//name"
Scenario: validate_namespaced_actor rejects special characters
Then validate_namespaced_actor rejects "ns@special/name"
And validate_namespaced_actor rejects "ns/name#bad"
Scenario: validate_namespaced_actor rejects numeric-leading namespace
Then validate_namespaced_actor rejects "1abc/model"
# ==========================================================================
# DEEP COVERAGE: Output snapshot assertions JSON/YAML/table (#326)
# ==========================================================================
Scenario: Plan status in JSON format contains required keys
Given cli extensions plans exist with automation profile "trusted"
When I run cli extensions plan status with format "json"
Then the cli extensions output is valid JSON
And the cli extensions json output contains key "automation_profile"
And the cli extensions json output contains key "namespaced_name"
And the cli extensions json output contains key "phase"
Scenario: Plan status in YAML format contains required fields
Given cli extensions plans exist with automation profile "trusted"
When I run cli extensions plan status with format "yaml"
Then the cli extensions output contains yaml field "automation_profile"
And the cli extensions output contains yaml field "namespaced_name"
Scenario: Action show in JSON format contains all required keys
Given a cli extensions action with estimation actor "openai/gpt-4"
When I run cli extensions action show with format "json"
Then the cli extensions output is valid JSON
And the cli extensions json output contains key "estimation_actor"
And the cli extensions json output contains key "strategy_actor"
And the cli extensions json output contains key "execution_actor"
And the cli extensions json output contains key "description"
Scenario: Action show in YAML format contains required fields
Given a cli extensions action with estimation actor "openai/gpt-4"
When I run cli extensions action show with format "yaml"
Then the cli extensions output contains yaml field "estimation_actor"
And the cli extensions output contains yaml field "strategy_actor"
Scenario: Action show in JSON includes invariants list when present
Given a cli extensions action with invariants
When I run cli extensions action show with format "json"
Then the cli extensions output is valid JSON
And the cli extensions json output contains key "invariants"
And the cli extensions json invariants list has length 2
Scenario: Plan lifecycle-list in YAML shows invariant data
Given cli extensions plans exist with invariants
When I run cli extensions plan lifecycle-list with format "yaml"
Then the cli extensions output contains yaml field "invariants"
Scenario: Plan lifecycle-list in JSON preserves invariant text
Given cli extensions plans exist with invariants
When I run cli extensions plan lifecycle-list with format "json"
Then the cli extensions output is valid JSON
And the cli extensions json output string contains "No warnings"
And the cli extensions json output string contains "Keep compat"
Scenario: Action show in table format contains actor information
Given a cli extensions action with estimation actor "openai/gpt-4"
When I run cli extensions action show with format "table"
Then the cli extensions action output should contain "openai/gpt-4"
Scenario: Plan status in table format contains profile name
Given cli extensions plans exist with automation profile "cautious"
When I run cli extensions plan status
Then the cli extensions status output should contain "cautious"
+183
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import json
from datetime import datetime
from unittest.mock import MagicMock, patch
@@ -621,6 +622,188 @@ def step_validate_actor_rejects(context: Context, actor: str) -> None:
pass # Expected
# ---------------------------------------------------------------------------
# DEEP COVERAGE: Automation profile resolution (#326)
# ---------------------------------------------------------------------------
@then("the cli extensions error output mentions available profiles")
def step_error_mentions_available(context: Context) -> None:
"""Verify error output lists available automation profiles."""
output = context.last_result.output
# The error message should mention at least some builtin profiles
assert "Available" in output or "manual" in output or "trusted" in output, (
f"Expected available profiles in error output: {output}"
)
# ---------------------------------------------------------------------------
# DEEP COVERAGE: Invariant ordering (#326)
# ---------------------------------------------------------------------------
@when('I run plan use with three invariants "{inv1}" and "{inv2}" and "{inv3}"')
def step_run_plan_use_three_invariants(
context: Context, inv1: str, inv2: str, inv3: str
) -> None:
"""Run plan use with three --invariant flags."""
result = context.runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--invariant",
inv1,
"--invariant",
inv2,
"--invariant",
inv3,
],
)
context.last_result = result
@then(
"the cli extensions service received invariants in order "
'"{inv1}" then "{inv2}" then "{inv3}"'
)
def step_service_received_ordered_invariants(
context: Context, inv1: str, inv2: str, inv3: str
) -> None:
"""Verify invariants were passed in exact insertion order."""
call_args = context.mock_service.use_action.call_args
assert call_args is not None, "use_action was not called"
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
assert invariants is not None, "No invariants passed to use_action"
texts = [inv.text for inv in invariants]
assert len(texts) >= 3, f"Expected at least 3 invariants, got {len(texts)}"
assert texts[0] == inv1, f"Expected '{inv1}' at index 0, got '{texts[0]}'"
assert texts[1] == inv2, f"Expected '{inv2}' at index 1, got '{texts[1]}'"
assert texts[2] == inv3, f"Expected '{inv3}' at index 2, got '{texts[2]}'"
@when(
"I run plan use with five invariants "
'"{inv1}" and "{inv2}" and "{inv3}" and "{inv4}" and "{inv5}"'
)
def step_run_plan_use_five_invariants(
context: Context,
inv1: str,
inv2: str,
inv3: str,
inv4: str,
inv5: str,
) -> None:
"""Run plan use with five --invariant flags."""
result = context.runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--invariant",
inv1,
"--invariant",
inv2,
"--invariant",
inv3,
"--invariant",
inv4,
"--invariant",
inv5,
],
)
context.last_result = result
@then(
"the cli extensions service received five invariants in exact order "
'"{inv1}" "{inv2}" "{inv3}" "{inv4}" "{inv5}"'
)
def step_service_received_five_ordered(
context: Context,
inv1: str,
inv2: str,
inv3: str,
inv4: str,
inv5: str,
) -> None:
"""Verify five invariants in exact insertion order."""
call_args = context.mock_service.use_action.call_args
assert call_args is not None, "use_action was not called"
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
assert invariants is not None, "No invariants passed"
texts = [inv.text for inv in invariants]
expected = [inv1, inv2, inv3, inv4, inv5]
assert texts == expected, f"Expected {expected}, got {texts}"
# ---------------------------------------------------------------------------
# DEEP COVERAGE: Output snapshot assertions (#326)
# ---------------------------------------------------------------------------
@then("the cli extensions output is valid JSON")
def step_output_is_valid_json(context: Context) -> None:
"""Verify last result output is valid JSON."""
assert context.last_result.exit_code == 0, (
f"Exit code: {context.last_result.exit_code}, "
f"output: {context.last_result.output}"
)
output = context.last_result.output.strip()
try:
context.parsed_json = json.loads(output)
except json.JSONDecodeError as exc:
raise AssertionError(
f"Output is not valid JSON: {exc}\nOutput: {output}"
) from exc
@then('the cli extensions json output contains key "{key}"')
def step_json_output_contains_key(context: Context, key: str) -> None:
"""Verify parsed JSON contains a specific key."""
parsed = getattr(context, "parsed_json", None)
if parsed is None:
# Parse if not already parsed
output = context.last_result.output.strip()
parsed = json.loads(output)
# Handle both dict and list-of-dict cases
if isinstance(parsed, list):
assert any(key in item for item in parsed if isinstance(item, dict)), (
f"Key '{key}' not found in any list item"
)
else:
assert key in parsed, f"Key '{key}' not in JSON: {list(parsed.keys())}"
@then('the cli extensions output contains yaml field "{field}"')
def step_output_contains_yaml_field(context: Context, field: str) -> None:
"""Verify output contains a YAML field (key: value pattern)."""
assert context.last_result.exit_code == 0
output = context.last_result.output
assert field in output, f"Expected YAML field '{field}' in output: {output}"
@then("the cli extensions json invariants list has length {length:d}")
def step_json_invariants_length(context: Context, length: int) -> None:
"""Verify JSON invariants list has expected length."""
parsed = getattr(context, "parsed_json", None)
if parsed is None:
output = context.last_result.output.strip()
parsed = json.loads(output)
invariants = parsed.get("invariants", [])
assert len(invariants) == length, (
f"Expected {length} invariants, got {len(invariants)}: {invariants}"
)
@then('the cli extensions json output string contains "{text}"')
def step_json_output_string_contains(context: Context, text: str) -> None:
"""Verify the raw JSON output string contains the given text."""
assert context.last_result.exit_code == 0
output = context.last_result.output
assert text in output, f"Expected '{text}' in JSON output: {output}"
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
+32
View File
@@ -47,3 +47,35 @@ Plan Use Combined Profile And Invariants
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cli-ext-plan-combined-ok
Action Show With Optional Actors And Invariants
[Documentation] Verify action show includes optional actors, invariants, and inputs_schema when set
${result}= Run Process ${PYTHON} ${HELPER} action-show-extended cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cli-ext-action-show-extended-ok
Action Show JSON With Optional Fields
[Documentation] Verify action show JSON output includes optional fields
${result}= Run Process ${PYTHON} ${HELPER} action-show-json cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cli-ext-action-show-json-ok
Invariant Ordering Preserved
[Documentation] Verify that multiple invariant flags preserve insertion order
${result}= Run Process ${PYTHON} ${HELPER} invariant-ordering cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cli-ext-invariant-ordering-ok
Profile Resolution All Builtins
[Documentation] Verify that all builtin profiles resolve correctly
${result}= Run Process ${PYTHON} ${HELPER} profile-resolution cwd=${WORKSPACE}
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cli-ext-profile-resolution-ok
+176
View File
@@ -17,6 +17,7 @@ if _SRC not in sys.path:
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.action import app as action_app # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.domain.models.core.action import ( # noqa: E402
Action,
@@ -62,6 +63,9 @@ def _mock_plan(
invariants=invariants or [],
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
created_by=None,
timestamps=PlanTimestamps(created_at=now, updated_at=now),
)
@@ -70,10 +74,14 @@ def _mock_action() -> Action:
return Action(
namespaced_name=NamespacedName.parse("local/test-action"),
description="Test action for robot",
long_description=None,
definition_of_done="All tests pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
@@ -228,6 +236,170 @@ def plan_use_combined() -> None:
sys.exit(1)
def _mock_action_extended() -> Action:
"""Create an action with all optional fields set for extended show tests."""
return Action(
namespaced_name=NamespacedName.parse("local/test-action"),
description="Test action for robot extended",
long_description=None,
definition_of_done="All tests pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor="openai/gpt-4",
invariant_actor="anthropic/claude-3",
reusable=True,
read_only=False,
invariants=["No regressions", "Keep backward compat"],
inputs_schema={
"type": "object",
"properties": {"coverage": {"type": "integer"}},
},
automation_profile="trusted",
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
def action_show_extended() -> None:
"""Verify action show renders optional actors, invariants, inputs_schema."""
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action_extended()
with patch(
"cleveragents.cli.commands.action._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(action_app, ["show", "local/test-action"])
output = result.output
checks = [
("Estimation Actor" in output or "estimation_actor" in output),
("Invariant Actor" in output or "invariant_actor" in output),
("Invariants" in output or "invariants" in output),
("Inputs Schema" in output or "inputs_schema" in output),
]
if result.exit_code == 0 and all(checks):
print("cli-ext-action-show-extended-ok")
else:
print(f"FAIL: exit={result.exit_code} output={output}")
print(f"checks={checks}")
sys.exit(1)
def action_show_json() -> None:
"""Verify action show JSON output includes optional fields."""
import json
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action_extended()
with patch(
"cleveragents.cli.commands.action._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
action_app, ["show", "local/test-action", "--format", "json"]
)
if result.exit_code != 0:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
try:
data = json.loads(result.output.strip())
except json.JSONDecodeError as exc:
print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}")
sys.exit(1)
required_keys = [
"estimation_actor",
"invariant_actor",
"inputs_schema",
"invariants",
]
missing = [k for k in required_keys if k not in data]
if missing:
print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}")
sys.exit(1)
print("cli-ext-action-show-json-ok")
def invariant_ordering() -> None:
"""Verify that multiple invariant flags preserve insertion order."""
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action()
mock_svc.use_action.return_value = _mock_plan(
invariants=[
PlanInvariant(text="Alpha", source=InvariantSource.PLAN),
PlanInvariant(text="Beta", source=InvariantSource.PLAN),
PlanInvariant(text="Gamma", source=InvariantSource.PLAN),
]
)
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--invariant",
"Alpha",
"--invariant",
"Beta",
"--invariant",
"Gamma",
],
)
if result.exit_code != 0:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
# Verify the service received invariants in order
call_args = mock_svc.use_action.call_args
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
if invariants is None:
print("FAIL: no invariants passed to use_action")
sys.exit(1)
texts = [inv.text for inv in invariants]
if texts != ["Alpha", "Beta", "Gamma"]:
print(f"FAIL: ordering mismatch: {texts}")
sys.exit(1)
print("cli-ext-invariant-ordering-ok")
def profile_resolution() -> None:
"""Verify all builtin profiles resolve correctly."""
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
for profile_name in sorted(BUILTIN_PROFILES):
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action()
mock_svc.use_action.return_value = _mock_plan(
automation_profile=AutomationProfileRef(
profile_name=profile_name,
provenance=AutomationProfileProvenance.PLAN,
)
)
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--automation-profile",
profile_name,
],
)
if result.exit_code != 0:
print(
f"FAIL: profile '{profile_name}' "
f"exit={result.exit_code} "
f"output={result.output}"
)
sys.exit(1)
print("cli-ext-profile-resolution-ok")
# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------
@@ -238,6 +410,10 @@ _COMMANDS: dict[str, object] = {
"actor-valid": plan_use_actor_validation_valid,
"actor-invalid": plan_use_actor_validation_invalid,
"plan-combined": plan_use_combined,
"action-show-extended": action_show_extended,
"action-show-json": action_show_json,
"invariant-ordering": invariant_ordering,
"profile-resolution": profile_resolution,
}
if __name__ == "__main__":