freemo
f69224e7e9
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Failing after 15s
CI / quality (pull_request) Successful in 17s
CI / build (pull_request) Successful in 17s
CI / security (pull_request) Successful in 29s
CI / typecheck (pull_request) Successful in 31s
CI / coverage (pull_request) Has been skipped
CI / benchmark-regression (pull_request) Has been skipped
CI / integration_tests (pull_request) Successful in 3m44s
CI / unit_tests (pull_request) Successful in 7m4s
CI / docker (pull_request) Has been skipped
568 lines
20 KiB
Python
568 lines
20 KiB
Python
"""Step definitions for CLI extensions feature (plan + action)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
from cleveragents.cli.commands.action import app as action_app
|
|
from cleveragents.cli.commands.plan import app as plan_app
|
|
from cleveragents.cli.commands.plan import validate_namespaced_actor
|
|
from cleveragents.core.exceptions import ValidationError
|
|
from cleveragents.domain.models.core.action import Action, ActionState
|
|
from cleveragents.domain.models.core.plan import (
|
|
AutomationProfileProvenance,
|
|
AutomationProfileRef,
|
|
InvariantSource,
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanInvariant,
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
)
|
|
|
|
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
|
|
|
|
|
def _make_plan(
|
|
*,
|
|
name: str = "local/test-plan",
|
|
action_name: str = "local/test-action",
|
|
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
|
state: ProcessingState = ProcessingState.QUEUED,
|
|
project_links: list[ProjectLink] | None = None,
|
|
arguments: dict[str, object] | None = None,
|
|
automation_profile: AutomationProfileRef | None = None,
|
|
invariants: list[PlanInvariant] | None = None,
|
|
strategy_actor: str | None = "openai/gpt-4",
|
|
execution_actor: str | None = "openai/gpt-4",
|
|
estimation_actor: str | None = None,
|
|
invariant_actor: str | None = None,
|
|
) -> Plan:
|
|
"""Create a Plan instance for CLI extension tests."""
|
|
now = datetime.now()
|
|
return Plan(
|
|
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Test plan description",
|
|
definition_of_done="All tests pass",
|
|
action_name=action_name,
|
|
phase=phase,
|
|
processing_state=state,
|
|
project_links=project_links or [],
|
|
arguments=dict(arguments) if arguments else {},
|
|
arguments_order=[],
|
|
automation_profile=automation_profile,
|
|
invariants=invariants or [],
|
|
strategy_actor=strategy_actor,
|
|
execution_actor=execution_actor,
|
|
estimation_actor=estimation_actor,
|
|
invariant_actor=invariant_actor,
|
|
reusable=True,
|
|
read_only=False,
|
|
created_by=None,
|
|
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
|
)
|
|
|
|
|
|
def _make_action(
|
|
name: str = "local/test-action",
|
|
*,
|
|
estimation_actor: str | None = None,
|
|
invariant_actor: str | None = None,
|
|
invariants: list[str] | None = None,
|
|
inputs_schema: dict[str, object] | None = None,
|
|
automation_profile: str | None = None,
|
|
) -> Action:
|
|
"""Create an Action for CLI extension tests."""
|
|
return Action(
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Test action description",
|
|
long_description=None,
|
|
definition_of_done="All tests pass",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
estimation_actor=estimation_actor,
|
|
invariant_actor=invariant_actor,
|
|
reusable=True,
|
|
read_only=False,
|
|
state=ActionState.AVAILABLE,
|
|
automation_profile=automation_profile,
|
|
invariants=invariants or [],
|
|
inputs_schema=inputs_schema,
|
|
created_by=None,
|
|
created_at=datetime.now(),
|
|
updated_at=datetime.now(),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a cli extensions test runner")
|
|
def step_cli_ext_runner(context: Context) -> None:
|
|
"""Set up the CLI runner."""
|
|
context.runner = CliRunner()
|
|
|
|
|
|
@given("a cli extensions mocked lifecycle service")
|
|
def step_cli_ext_mock_service(context: Context) -> None:
|
|
"""Set up the mocked lifecycle service."""
|
|
context.mock_service = MagicMock()
|
|
context.plan_patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=context.mock_service,
|
|
)
|
|
context.action_patcher = patch(
|
|
"cleveragents.cli.commands.action._get_lifecycle_service",
|
|
return_value=context.mock_service,
|
|
)
|
|
context.plan_patcher.start()
|
|
context.action_patcher.start()
|
|
# Store the last plan created for inspection
|
|
context.last_plan = None
|
|
context.last_result = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: action setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a cli extensions action exists")
|
|
def step_cli_ext_action_exists(context: Context) -> None:
|
|
"""Set up an action for plan use tests."""
|
|
context.mock_service.get_action_by_name.return_value = _make_action()
|
|
context.mock_service.use_action.return_value = _make_plan()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: automation profile
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan use with automation profile flag "{profile}"')
|
|
def step_run_plan_use_with_profile(context: Context, profile: str) -> None:
|
|
"""Run plan use with --automation-profile."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--automation-profile", profile],
|
|
)
|
|
context.last_result = result
|
|
context.last_profile = profile
|
|
|
|
|
|
@then("the cli extensions plan use should succeed")
|
|
def step_plan_use_should_succeed(context: Context) -> None:
|
|
"""Verify plan use succeeded."""
|
|
assert context.last_result is not None
|
|
assert context.last_result.exit_code == 0, (
|
|
f"Expected exit code 0, got {context.last_result.exit_code}. "
|
|
f"Output: {context.last_result.output}"
|
|
)
|
|
|
|
|
|
@then("the cli extensions plan use should fail")
|
|
def step_plan_use_should_fail(context: Context) -> None:
|
|
"""Verify plan use failed."""
|
|
assert context.last_result is not None
|
|
assert context.last_result.exit_code != 0
|
|
|
|
|
|
@then('the cli extensions plan should have automation profile "{profile}"')
|
|
def step_plan_has_profile(context: Context, profile: str) -> None:
|
|
"""Verify plan output mentions automation profile."""
|
|
output = context.last_result.output
|
|
assert profile in output, f"Expected '{profile}' in output: {output}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: invariant flags
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan use with invariant flags "{inv1}" and "{inv2}"')
|
|
def step_run_plan_use_with_invariants(context: Context, inv1: str, inv2: str) -> None:
|
|
"""Run plan use with two --invariant flags."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/test-action",
|
|
"--invariant",
|
|
inv1,
|
|
"--invariant",
|
|
inv2,
|
|
],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@then('the cli extensions service received invariants "{inv1}" and "{inv2}"')
|
|
def step_service_received_invariants(context: Context, inv1: str, inv2: str) -> None:
|
|
"""Verify invariants were passed to the service."""
|
|
call_args = context.mock_service.use_action.call_args
|
|
assert call_args is not None, "use_action was not called"
|
|
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
|
assert invariants is not None, "No invariants passed to use_action"
|
|
texts = [inv.text for inv in invariants]
|
|
assert inv1 in texts, f"Expected '{inv1}' in {texts}"
|
|
assert inv2 in texts, f"Expected '{inv2}' in {texts}"
|
|
|
|
|
|
@when('I run plan use with single invariant "{inv_text}"')
|
|
def step_run_plan_use_with_single_invariant(context: Context, inv_text: str) -> None:
|
|
"""Run plan use with a single --invariant flag."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--invariant", inv_text],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@then('the cli extensions service received single invariant "{inv_text}"')
|
|
def step_service_received_single_invariant(context: Context, inv_text: str) -> None:
|
|
"""Verify single invariant was passed."""
|
|
call_args = context.mock_service.use_action.call_args
|
|
assert call_args is not None
|
|
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
|
assert invariants is not None
|
|
texts = [inv.text for inv in invariants]
|
|
assert inv_text in texts, f"Expected '{inv_text}' in {texts}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: actor override validation (valid)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan use with strategy actor flag "{actor}"')
|
|
def step_run_plan_use_strategy_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --strategy-actor."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--strategy-actor", actor],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@when('I run plan use with execution actor flag "{actor}"')
|
|
def step_run_plan_use_execution_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --execution-actor."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--execution-actor", actor],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@when('I run plan use with estimation actor flag "{actor}"')
|
|
def step_run_plan_use_estimation_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --estimation-actor."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--estimation-actor", actor],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@when('I run plan use with invariant actor flag "{actor}"')
|
|
def step_run_plan_use_invariant_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --invariant-actor."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "--invariant-actor", actor],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@then('the cli extensions plan has strategy actor "{actor}"')
|
|
def step_plan_has_strategy_actor(context: Context, actor: str) -> None:
|
|
"""Verify plan output shows the strategy actor."""
|
|
output = context.last_result.output
|
|
assert actor in output, f"Expected '{actor}' in output: {output}"
|
|
|
|
|
|
@then('the cli extensions plan has execution actor "{actor}"')
|
|
def step_plan_has_execution_actor(context: Context, actor: str) -> None:
|
|
"""Verify plan output shows the execution actor."""
|
|
output = context.last_result.output
|
|
assert actor in output, f"Expected '{actor}' in output: {output}"
|
|
|
|
|
|
@then('the cli extensions plan has estimation actor "{actor}"')
|
|
def step_plan_has_estimation_actor(context: Context, actor: str) -> None:
|
|
"""Verify plan output shows the estimation actor."""
|
|
output = context.last_result.output
|
|
assert actor in output, f"Expected '{actor}' in output: {output}"
|
|
|
|
|
|
@then('the cli extensions plan has invariant actor "{actor}"')
|
|
def step_plan_has_invariant_actor(context: Context, actor: str) -> None:
|
|
"""Verify plan output shows the invariant actor."""
|
|
output = context.last_result.output
|
|
assert actor in output, f"Expected '{actor}' in output: {output}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: actor override validation (invalid format)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the cli extensions plan use should fail with actor validation error")
|
|
def step_plan_use_fail_actor_validation(context: Context) -> None:
|
|
"""Verify plan use failed due to actor validation."""
|
|
assert context.last_result is not None
|
|
assert context.last_result.exit_code != 0, (
|
|
f"Expected non-zero exit, got {context.last_result.exit_code}. "
|
|
f"Output: {context.last_result.output}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan status: automation profile
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('cli extensions plans exist with automation profile "{profile}"')
|
|
def step_plans_with_profile(context: Context, profile: str) -> None:
|
|
"""Set up plans that have an automation profile."""
|
|
plan = _make_plan(
|
|
automation_profile=AutomationProfileRef(
|
|
profile_name=profile,
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
)
|
|
)
|
|
context.mock_service.list_plans.return_value = [plan]
|
|
context.mock_service.get_plan.return_value = plan
|
|
|
|
|
|
@when("I run cli extensions plan status")
|
|
def step_run_plan_status(context: Context) -> None:
|
|
"""Run plan status (no plan_id = list all)."""
|
|
result = context.runner.invoke(plan_app, ["status"])
|
|
context.last_result = result
|
|
|
|
|
|
@then('the cli extensions status output should contain "{text}"')
|
|
def step_status_contains(context: Context, text: str) -> None:
|
|
"""Verify status output contains text."""
|
|
assert context.last_result.exit_code == 0
|
|
assert text in context.last_result.output, (
|
|
f"Expected '{text}' in: {context.last_result.output}"
|
|
)
|
|
|
|
|
|
@when('I run cli extensions plan status with format "{fmt}"')
|
|
def step_run_plan_status_fmt(context: Context, fmt: str) -> None:
|
|
"""Run plan status with --format."""
|
|
result = context.runner.invoke(plan_app, ["status", "--format", fmt])
|
|
context.last_result = result
|
|
|
|
|
|
@then(
|
|
"the cli extensions status json output should contain automation profile "
|
|
'"{profile}"'
|
|
)
|
|
def step_status_json_profile(context: Context, profile: str) -> None:
|
|
"""Verify JSON output contains automation profile."""
|
|
assert context.last_result.exit_code == 0
|
|
output = context.last_result.output
|
|
assert profile in output, f"Expected '{profile}' in: {output}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan lifecycle-list: invariants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("cli extensions plans exist with invariants")
|
|
def step_plans_with_invariants(context: Context) -> None:
|
|
"""Set up plans that have invariants."""
|
|
plan = _make_plan(
|
|
invariants=[
|
|
PlanInvariant(text="No warnings", source=InvariantSource.PLAN),
|
|
PlanInvariant(text="Keep compat", source=InvariantSource.ACTION),
|
|
]
|
|
)
|
|
context.mock_service.list_plans.return_value = [plan]
|
|
|
|
|
|
@when("I run cli extensions plan lifecycle-list")
|
|
def step_run_lifecycle_list(context: Context) -> None:
|
|
"""Run plan lifecycle-list."""
|
|
result = context.runner.invoke(plan_app, ["lifecycle-list"])
|
|
context.last_result = result
|
|
|
|
|
|
@then("the cli extensions list output should contain invariant count")
|
|
def step_list_has_invariant_count(context: Context) -> None:
|
|
"""Verify list output contains invariant count."""
|
|
assert context.last_result.exit_code == 0
|
|
# The table should show "2" for invariant count
|
|
assert "2" in context.last_result.output, (
|
|
f"Expected '2' invariant count in: {context.last_result.output}"
|
|
)
|
|
|
|
|
|
@when('I run cli extensions plan lifecycle-list with format "{fmt}"')
|
|
def step_run_lifecycle_list_fmt(context: Context, fmt: str) -> None:
|
|
"""Run plan lifecycle-list with --format."""
|
|
result = context.runner.invoke(plan_app, ["lifecycle-list", "--format", fmt])
|
|
context.last_result = result
|
|
|
|
|
|
@then("the cli extensions list json output should contain invariants")
|
|
def step_list_json_invariants(context: Context) -> None:
|
|
"""Verify JSON output contains invariants field."""
|
|
assert context.last_result.exit_code == 0
|
|
output = context.last_result.output
|
|
assert "invariants" in output, f"Expected 'invariants' in: {output}"
|
|
assert "No warnings" in output, f"Expected 'No warnings' in: {output}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# action show: optional fields
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given('a cli extensions action with estimation actor "{actor}"')
|
|
def step_action_with_estimation_actor(context: Context, actor: str) -> None:
|
|
"""Set up an action with estimation actor."""
|
|
action = _make_action(estimation_actor=actor)
|
|
context.mock_service.get_action_by_name.return_value = action
|
|
context.test_action = action
|
|
|
|
|
|
@given('a cli extensions action with invariant actor "{actor}"')
|
|
def step_action_with_invariant_actor(context: Context, actor: str) -> None:
|
|
"""Set up an action with invariant actor."""
|
|
action = _make_action(invariant_actor=actor)
|
|
context.mock_service.get_action_by_name.return_value = action
|
|
context.test_action = action
|
|
|
|
|
|
@given("a cli extensions action with invariants")
|
|
def step_action_with_invariants(context: Context) -> None:
|
|
"""Set up an action with invariants."""
|
|
action = _make_action(invariants=["No regressions", "Keep backward compat"])
|
|
context.mock_service.get_action_by_name.return_value = action
|
|
context.test_action = action
|
|
|
|
|
|
@given("a cli extensions action with inputs schema")
|
|
def step_action_with_inputs_schema(context: Context) -> None:
|
|
"""Set up an action with inputs_schema."""
|
|
action = _make_action(
|
|
inputs_schema={
|
|
"type": "object",
|
|
"properties": {"coverage": {"type": "integer"}},
|
|
}
|
|
)
|
|
context.mock_service.get_action_by_name.return_value = action
|
|
context.test_action = action
|
|
|
|
|
|
@when("I run cli extensions action show")
|
|
def step_run_action_show(context: Context) -> None:
|
|
"""Run action show."""
|
|
result = context.runner.invoke(action_app, ["show", "local/test-action"])
|
|
context.last_result = result
|
|
|
|
|
|
@when('I run cli extensions action show with format "{fmt}"')
|
|
def step_run_action_show_fmt(context: Context, fmt: str) -> None:
|
|
"""Run action show with --format."""
|
|
result = context.runner.invoke(
|
|
action_app, ["show", "local/test-action", "--format", fmt]
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
@then('the cli extensions action output should contain "{text}"')
|
|
def step_action_output_contains(context: Context, text: str) -> None:
|
|
"""Verify action output contains text."""
|
|
assert context.last_result.exit_code == 0, (
|
|
f"Exit code: {context.last_result.exit_code}, "
|
|
f"output: {context.last_result.output}"
|
|
)
|
|
assert text in context.last_result.output, (
|
|
f"Expected '{text}' in: {context.last_result.output}"
|
|
)
|
|
|
|
|
|
@then('the cli extensions action json should contain "{key}"')
|
|
def step_action_json_contains(context: Context, key: str) -> None:
|
|
"""Verify JSON output contains key."""
|
|
assert context.last_result.exit_code == 0
|
|
output = context.last_result.output
|
|
assert key in output, f"Expected '{key}' in: {output}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# plan use: combined profile + invariant
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan use with profile "{profile}" and invariant "{inv_text}"')
|
|
def step_run_plan_use_profile_invariant(
|
|
context: Context, profile: str, inv_text: str
|
|
) -> None:
|
|
"""Run plan use with both --automation-profile and --invariant."""
|
|
result = context.runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/test-action",
|
|
"--automation-profile",
|
|
profile,
|
|
"--invariant",
|
|
inv_text,
|
|
],
|
|
)
|
|
context.last_result = result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_namespaced_actor unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then('validate_namespaced_actor accepts "{actor}"')
|
|
def step_validate_actor_accepts(context: Context, actor: str) -> None:
|
|
"""Verify validate_namespaced_actor accepts a valid name."""
|
|
result = validate_namespaced_actor(actor, "--test")
|
|
assert result == actor
|
|
|
|
|
|
@then('validate_namespaced_actor rejects "{actor}"')
|
|
def step_validate_actor_rejects(context: Context, actor: str) -> None:
|
|
"""Verify validate_namespaced_actor rejects an invalid name."""
|
|
try:
|
|
validate_namespaced_actor(actor, "--test")
|
|
raise AssertionError(f"Expected ValidationError for '{actor}'")
|
|
except ValidationError:
|
|
pass # Expected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cleanup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def after_scenario(context: Context, scenario: object) -> None:
|
|
"""Clean up patchers."""
|
|
for name in ("plan_patcher", "action_patcher"):
|
|
patcher = getattr(context, name, None)
|
|
if patcher:
|
|
patcher.stop()
|