test(cli): cover action and plan extensions
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s
CI / lint (pull_request) Successful in 25s
CI / quality (pull_request) Successful in 37s
CI / benchmark-publish (pull_request) Has been skipped
CI / security (pull_request) Successful in 55s
CI / typecheck (pull_request) Successful in 1m1s
CI / build (pull_request) Successful in 33s
CI / integration_tests (pull_request) Successful in 5m13s
CI / unit_tests (pull_request) Successful in 12m53s
CI / docker (pull_request) Successful in 9s
CI / benchmark-regression (pull_request) Successful in 21m31s
CI / coverage (pull_request) Successful in 51m7s
Added comprehensive test coverage for CLI extension features from #325: - Behave scenarios for automation_profile resolution, invariant ordering, and actor override error cases - Output snapshot assertions for JSON/YAML/table formats - Robot integration test for action show with optional actors/invariants - ASV benchmark for extended CLI scenario runtime baseline - Updated testing.md with CLI extension test fixture documentation ISSUES CLOSED: #326
This commit is contained in:
@@ -17,6 +17,7 @@ if _SRC not in sys.path:
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
@@ -62,6 +63,9 @@ def _mock_plan(
|
||||
invariants=invariants or [],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
@@ -70,10 +74,14 @@ def _mock_action() -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
@@ -228,6 +236,170 @@ def plan_use_combined() -> None:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _mock_action_extended() -> Action:
|
||||
"""Create an action with all optional fields set for extended show tests."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot extended",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
automation_profile="trusted",
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def action_show_extended() -> None:
|
||||
"""Verify action show renders optional actors, invariants, inputs_schema."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["show", "local/test-action"])
|
||||
output = result.output
|
||||
checks = [
|
||||
("Estimation Actor" in output or "estimation_actor" in output),
|
||||
("Invariant Actor" in output or "invariant_actor" in output),
|
||||
("Invariants" in output or "invariants" in output),
|
||||
("Inputs Schema" in output or "inputs_schema" in output),
|
||||
]
|
||||
if result.exit_code == 0 and all(checks):
|
||||
print("cli-ext-action-show-extended-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={output}")
|
||||
print(f"checks={checks}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def action_show_json() -> None:
|
||||
"""Verify action show JSON output includes optional fields."""
|
||||
import json
|
||||
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
action_app, ["show", "local/test-action", "--format", "json"]
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
try:
|
||||
data = json.loads(result.output.strip())
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}")
|
||||
sys.exit(1)
|
||||
required_keys = [
|
||||
"estimation_actor",
|
||||
"invariant_actor",
|
||||
"inputs_schema",
|
||||
"invariants",
|
||||
]
|
||||
missing = [k for k in required_keys if k not in data]
|
||||
if missing:
|
||||
print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-action-show-json-ok")
|
||||
|
||||
|
||||
def invariant_ordering() -> None:
|
||||
"""Verify that multiple invariant flags preserve insertion order."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
invariants=[
|
||||
PlanInvariant(text="Alpha", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Beta", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Gamma", source=InvariantSource.PLAN),
|
||||
]
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
"Alpha",
|
||||
"--invariant",
|
||||
"Beta",
|
||||
"--invariant",
|
||||
"Gamma",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
# Verify the service received invariants in order
|
||||
call_args = mock_svc.use_action.call_args
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
if invariants is None:
|
||||
print("FAIL: no invariants passed to use_action")
|
||||
sys.exit(1)
|
||||
texts = [inv.text for inv in invariants]
|
||||
if texts != ["Alpha", "Beta", "Gamma"]:
|
||||
print(f"FAIL: ordering mismatch: {texts}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-invariant-ordering-ok")
|
||||
|
||||
|
||||
def profile_resolution() -> None:
|
||||
"""Verify all builtin profiles resolve correctly."""
|
||||
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
|
||||
|
||||
for profile_name in sorted(BUILTIN_PROFILES):
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
automation_profile=AutomationProfileRef(
|
||||
profile_name=profile_name,
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--automation-profile",
|
||||
profile_name,
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(
|
||||
f"FAIL: profile '{profile_name}' "
|
||||
f"exit={result.exit_code} "
|
||||
f"output={result.output}"
|
||||
)
|
||||
sys.exit(1)
|
||||
print("cli-ext-profile-resolution-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -238,6 +410,10 @@ _COMMANDS: dict[str, object] = {
|
||||
"actor-valid": plan_use_actor_validation_valid,
|
||||
"actor-invalid": plan_use_actor_validation_invalid,
|
||||
"plan-combined": plan_use_combined,
|
||||
"action-show-extended": action_show_extended,
|
||||
"action-show-json": action_show_json,
|
||||
"invariant-ordering": invariant_ordering,
|
||||
"profile-resolution": profile_resolution,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user