test(cli): cover action and plan extensions
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s

Added comprehensive test coverage for CLI extension features from #325:
- Behave scenarios for automation_profile resolution, invariant ordering,
  and actor override error cases
- Output snapshot assertions for JSON/YAML/table formats
- Robot integration test for action show with optional actors/invariants
- ASV benchmark for extended CLI scenario runtime baseline
- Updated testing.md with CLI extension test fixture documentation

ISSUES CLOSED: #326
This commit is contained in:
2026-02-24 20:11:42 +00:00
parent c47e6445d0
commit 2c21611ad5
7 changed files with 1024 additions and 0 deletions
+176
View File
@@ -17,6 +17,7 @@ if _SRC not in sys.path:
from typer.testing import CliRunner # noqa: E402
from cleveragents.cli.commands.action import app as action_app # noqa: E402
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
from cleveragents.domain.models.core.action import ( # noqa: E402
Action,
@@ -62,6 +63,9 @@ def _mock_plan(
invariants=invariants or [],
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
created_by=None,
timestamps=PlanTimestamps(created_at=now, updated_at=now),
)
@@ -70,10 +74,14 @@ def _mock_action() -> Action:
return Action(
namespaced_name=NamespacedName.parse("local/test-action"),
description="Test action for robot",
long_description=None,
definition_of_done="All tests pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
reusable=True,
read_only=False,
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
@@ -228,6 +236,170 @@ def plan_use_combined() -> None:
sys.exit(1)
def _mock_action_extended() -> Action:
"""Create an action with all optional fields set for extended show tests."""
return Action(
namespaced_name=NamespacedName.parse("local/test-action"),
description="Test action for robot extended",
long_description=None,
definition_of_done="All tests pass",
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
estimation_actor="openai/gpt-4",
invariant_actor="anthropic/claude-3",
reusable=True,
read_only=False,
invariants=["No regressions", "Keep backward compat"],
inputs_schema={
"type": "object",
"properties": {"coverage": {"type": "integer"}},
},
automation_profile="trusted",
state=ActionState.AVAILABLE,
created_by=None,
created_at=datetime.now(),
updated_at=datetime.now(),
)
def action_show_extended() -> None:
"""Verify action show renders optional actors, invariants, inputs_schema."""
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action_extended()
with patch(
"cleveragents.cli.commands.action._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(action_app, ["show", "local/test-action"])
output = result.output
checks = [
("Estimation Actor" in output or "estimation_actor" in output),
("Invariant Actor" in output or "invariant_actor" in output),
("Invariants" in output or "invariants" in output),
("Inputs Schema" in output or "inputs_schema" in output),
]
if result.exit_code == 0 and all(checks):
print("cli-ext-action-show-extended-ok")
else:
print(f"FAIL: exit={result.exit_code} output={output}")
print(f"checks={checks}")
sys.exit(1)
def action_show_json() -> None:
"""Verify action show JSON output includes optional fields."""
import json
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action_extended()
with patch(
"cleveragents.cli.commands.action._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
action_app, ["show", "local/test-action", "--format", "json"]
)
if result.exit_code != 0:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
try:
data = json.loads(result.output.strip())
except json.JSONDecodeError as exc:
print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}")
sys.exit(1)
required_keys = [
"estimation_actor",
"invariant_actor",
"inputs_schema",
"invariants",
]
missing = [k for k in required_keys if k not in data]
if missing:
print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}")
sys.exit(1)
print("cli-ext-action-show-json-ok")
def invariant_ordering() -> None:
"""Verify that multiple invariant flags preserve insertion order."""
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action()
mock_svc.use_action.return_value = _mock_plan(
invariants=[
PlanInvariant(text="Alpha", source=InvariantSource.PLAN),
PlanInvariant(text="Beta", source=InvariantSource.PLAN),
PlanInvariant(text="Gamma", source=InvariantSource.PLAN),
]
)
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--invariant",
"Alpha",
"--invariant",
"Beta",
"--invariant",
"Gamma",
],
)
if result.exit_code != 0:
print(f"FAIL: exit={result.exit_code} output={result.output}")
sys.exit(1)
# Verify the service received invariants in order
call_args = mock_svc.use_action.call_args
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
if invariants is None:
print("FAIL: no invariants passed to use_action")
sys.exit(1)
texts = [inv.text for inv in invariants]
if texts != ["Alpha", "Beta", "Gamma"]:
print(f"FAIL: ordering mismatch: {texts}")
sys.exit(1)
print("cli-ext-invariant-ordering-ok")
def profile_resolution() -> None:
"""Verify all builtin profiles resolve correctly."""
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
for profile_name in sorted(BUILTIN_PROFILES):
mock_svc = MagicMock()
mock_svc.get_action_by_name.return_value = _mock_action()
mock_svc.use_action.return_value = _mock_plan(
automation_profile=AutomationProfileRef(
profile_name=profile_name,
provenance=AutomationProfileProvenance.PLAN,
)
)
with patch(
"cleveragents.cli.commands.plan._get_lifecycle_service",
return_value=mock_svc,
):
result = runner.invoke(
plan_app,
[
"use",
"local/test-action",
"--automation-profile",
profile_name,
],
)
if result.exit_code != 0:
print(
f"FAIL: profile '{profile_name}' "
f"exit={result.exit_code} "
f"output={result.output}"
)
sys.exit(1)
print("cli-ext-profile-resolution-ok")
# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------
@@ -238,6 +410,10 @@ _COMMANDS: dict[str, object] = {
"actor-valid": plan_use_actor_validation_valid,
"actor-invalid": plan_use_actor_validation_invalid,
"plan-combined": plan_use_combined,
"action-show-extended": action_show_extended,
"action-show-json": action_show_json,
"invariant-ordering": invariant_ordering,
"profile-resolution": profile_resolution,
}
if __name__ == "__main__":