forked from HAL9000/cleveragents-core
459 lines
16 KiB
Python
459 lines
16 KiB
Python
"""Step definitions for plan CLI spec alignment feature."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
from typer.testing import CliRunner
|
|
|
|
from cleveragents.cli.commands.plan import app as plan_app
|
|
from cleveragents.domain.models.core.action import Action, ActionState
|
|
from cleveragents.domain.models.core.plan import (
|
|
AutomationProfileProvenance,
|
|
AutomationProfileRef,
|
|
NamespacedName,
|
|
Plan,
|
|
PlanIdentity,
|
|
PlanInvariant,
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
ProjectLink,
|
|
)
|
|
|
|
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
|
|
|
|
|
def _make_plan(
|
|
*,
|
|
name: str = "local/test-plan",
|
|
action_name: str = "local/test-action",
|
|
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
|
state: ProcessingState = ProcessingState.QUEUED,
|
|
project_links: list[ProjectLink] | None = None,
|
|
arguments: dict[str, object] | None = None,
|
|
arguments_order: list[str] | None = None,
|
|
automation_profile: AutomationProfileRef | None = None,
|
|
invariants: list[PlanInvariant] | None = None,
|
|
strategy_actor: str | None = "openai/gpt-4",
|
|
execution_actor: str | None = "openai/gpt-4",
|
|
estimation_actor: str | None = None,
|
|
invariant_actor: str | None = None,
|
|
) -> Plan:
|
|
"""Create a Plan instance for spec alignment tests."""
|
|
now = datetime.now()
|
|
return Plan(
|
|
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Test plan description",
|
|
definition_of_done="All tests pass",
|
|
action_name=action_name,
|
|
phase=phase,
|
|
processing_state=state,
|
|
project_links=project_links or [],
|
|
arguments=dict(arguments) if arguments else {},
|
|
arguments_order=arguments_order or [],
|
|
automation_profile=automation_profile,
|
|
invariants=invariants or [],
|
|
strategy_actor=strategy_actor,
|
|
execution_actor=execution_actor,
|
|
estimation_actor=estimation_actor,
|
|
invariant_actor=invariant_actor,
|
|
reusable=True,
|
|
read_only=False,
|
|
created_by=None,
|
|
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
|
)
|
|
|
|
|
|
def _make_action(name: str = "local/test-action") -> Action:
|
|
"""Create an Action for plan use tests."""
|
|
return Action(
|
|
namespaced_name=NamespacedName.parse(name),
|
|
description="Test action",
|
|
long_description=None,
|
|
definition_of_done="All tests pass",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
reusable=True,
|
|
read_only=False,
|
|
state=ActionState.AVAILABLE,
|
|
created_by=None,
|
|
created_at=datetime.now(),
|
|
updated_at=datetime.now(),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a plan spec alignment CLI runner")
|
|
def step_plan_spec_runner(context: Context) -> None:
|
|
"""Set up the CLI runner."""
|
|
context.runner = CliRunner()
|
|
|
|
|
|
@given("a plan spec alignment mocked lifecycle service")
|
|
def step_plan_spec_service(context: Context) -> None:
|
|
"""Set up a mock PlanLifecycleService."""
|
|
context.mock_service = MagicMock()
|
|
context.service_patcher = patch(
|
|
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
|
return_value=context.mock_service,
|
|
)
|
|
context.service_patcher.start()
|
|
if not hasattr(context, "_cleanup_handlers"):
|
|
context._cleanup_handlers = []
|
|
context._cleanup_handlers.append(context.service_patcher.stop)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Given steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a plan spec alignment action exists")
|
|
def step_plan_spec_action(context: Context) -> None:
|
|
"""Set up an action and configure mock service for use_action."""
|
|
context.mock_action = _make_action()
|
|
context.mock_service.get_action_by_name.return_value = context.mock_action
|
|
# Default plan returned by use_action
|
|
context.mock_plan = _make_plan(
|
|
project_links=[ProjectLink(project_name="proj-a")],
|
|
arguments={"target_coverage": 80},
|
|
arguments_order=["target_coverage"],
|
|
automation_profile=AutomationProfileRef(
|
|
profile_name="trusted",
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
),
|
|
estimation_actor="openai/gpt-4",
|
|
invariant_actor="openai/gpt-4",
|
|
)
|
|
context.mock_service.use_action.return_value = context.mock_plan
|
|
|
|
|
|
@given("plan spec alignment plans exist")
|
|
def step_plan_spec_plans(context: Context) -> None:
|
|
"""Set up plans for list filtering tests."""
|
|
context.mock_plans = [
|
|
_make_plan(
|
|
name="local/plan-a",
|
|
action_name="local/test-action",
|
|
project_links=[ProjectLink(project_name="proj-a")],
|
|
),
|
|
_make_plan(
|
|
name="local/plan-b",
|
|
action_name="local/other-action",
|
|
phase=PlanPhase.EXECUTE,
|
|
state=ProcessingState.COMPLETE,
|
|
project_links=[ProjectLink(project_name="proj-b")],
|
|
),
|
|
]
|
|
context.mock_service.list_plans.return_value = context.mock_plans
|
|
|
|
|
|
@given("a plan spec alignment plan exists for status")
|
|
def step_plan_spec_status_plan(context: Context) -> None:
|
|
"""Set up a plan with all fields for status rendering."""
|
|
context.mock_plan = _make_plan(
|
|
project_links=[
|
|
ProjectLink(project_name="proj-a", alias="api"),
|
|
ProjectLink(project_name="proj-b", read_only=True),
|
|
],
|
|
arguments={"coverage": 80, "framework": "behave"},
|
|
arguments_order=["coverage", "framework"],
|
|
automation_profile=AutomationProfileRef(
|
|
profile_name="trusted",
|
|
provenance=AutomationProfileProvenance.PLAN,
|
|
),
|
|
estimation_actor="openai/gpt-4",
|
|
invariant_actor="openai/gpt-4",
|
|
)
|
|
context.mock_service.get_plan.return_value = context.mock_plan
|
|
|
|
|
|
@given("a plan spec alignment plan exists for cancel")
|
|
def step_plan_spec_cancel_plan(context: Context) -> None:
|
|
"""Set up a plan for cancel tests."""
|
|
context.mock_plan = _make_plan()
|
|
context.mock_plan.processing_state = ProcessingState.CANCELLED
|
|
context.mock_service.cancel_plan.return_value = context.mock_plan
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps -- plan use
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan use with positional projects "{proj_a}" and "{proj_b}"')
|
|
def step_plan_use_positional(context: Context, proj_a: str, proj_b: str) -> None:
|
|
"""Run plan use with multiple positional project args."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["use", "local/test-action", proj_a, proj_b]
|
|
)
|
|
|
|
|
|
@when('I run plan use with automation profile "{profile}"')
|
|
def step_plan_use_profile(context: Context, profile: str) -> None:
|
|
"""Run plan use with --automation-profile."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/test-action",
|
|
"proj-a",
|
|
"--automation-profile",
|
|
profile,
|
|
],
|
|
)
|
|
|
|
|
|
@when('I run plan use with invariants "{inv_a}" and "{inv_b}"')
|
|
def step_plan_use_with_invariants(context: Context, inv_a: str, inv_b: str) -> None:
|
|
"""Run plan use with repeatable --invariant flags."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
[
|
|
"use",
|
|
"local/test-action",
|
|
"proj-a",
|
|
"--invariant",
|
|
inv_a,
|
|
"--invariant",
|
|
inv_b,
|
|
],
|
|
)
|
|
|
|
|
|
@when('I run plan use with strategy actor "{actor}"')
|
|
def step_plan_use_strategy_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --strategy-actor."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "proj-a", "--strategy-actor", actor],
|
|
)
|
|
|
|
|
|
@when('I run plan use with execution actor "{actor}"')
|
|
def step_plan_use_execution_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --execution-actor."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "proj-a", "--execution-actor", actor],
|
|
)
|
|
|
|
|
|
@when('I run plan use with estimation actor "{actor}"')
|
|
def step_plan_use_estimation_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --estimation-actor."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "proj-a", "--estimation-actor", actor],
|
|
)
|
|
|
|
|
|
@when('I run plan use with invariant actor "{actor}"')
|
|
def step_plan_use_invariant_actor(context: Context, actor: str) -> None:
|
|
"""Run plan use with --invariant-actor."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "proj-a", "--invariant-actor", actor],
|
|
)
|
|
|
|
|
|
@when('I run plan use with arg "{arg_str}"')
|
|
def step_plan_use_arg(context: Context, arg_str: str) -> None:
|
|
"""Run plan use with --arg name=value."""
|
|
context.result = context.runner.invoke(
|
|
plan_app,
|
|
["use", "local/test-action", "proj-a", "--arg", arg_str],
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps -- plan lifecycle-list
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan lifecycle-list with phase "{phase}"')
|
|
def step_plan_list_phase(context: Context, phase: str) -> None:
|
|
"""Run lifecycle-list with --phase filter."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--phase", phase]
|
|
)
|
|
|
|
|
|
@when('I run plan lifecycle-list with state "{state}"')
|
|
def step_plan_list_state(context: Context, state: str) -> None:
|
|
"""Run lifecycle-list with --state filter."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--state", state]
|
|
)
|
|
|
|
|
|
@when('I run plan lifecycle-list with processing-state "{state}"')
|
|
def step_plan_list_processing_state(context: Context, state: str) -> None:
|
|
"""Run lifecycle-list with --processing-state filter."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--processing-state", state]
|
|
)
|
|
|
|
|
|
@when('I run plan lifecycle-list with project "{project}"')
|
|
def step_plan_list_project(context: Context, project: str) -> None:
|
|
"""Run lifecycle-list with --project filter."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--project", project]
|
|
)
|
|
|
|
|
|
@when('I run plan lifecycle-list with action "{action}"')
|
|
def step_plan_list_action(context: Context, action: str) -> None:
|
|
"""Run lifecycle-list with --action filter."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--action", action]
|
|
)
|
|
|
|
|
|
@when('I run plan lifecycle-list with regex "{regex}"')
|
|
def step_plan_list_regex(context: Context, regex: str) -> None:
|
|
"""Run lifecycle-list with a regex positional argument."""
|
|
context.result = context.runner.invoke(plan_app, ["lifecycle-list", regex])
|
|
|
|
|
|
@when('I run plan lifecycle-list combining phase "{phase}" with project "{project}"')
|
|
def step_plan_list_combined(context: Context, phase: str, project: str) -> None:
|
|
"""Run lifecycle-list with combined filters."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["lifecycle-list", "--phase", phase, "--project", project]
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps -- plan status
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when("I run plan status for the plan")
|
|
def step_plan_status(context: Context) -> None:
|
|
"""Run plan status for a specific plan."""
|
|
context.result = context.runner.invoke(plan_app, ["status", _PLAN_ULID])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# When steps -- plan cancel
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@when('I run plan cancel with reason "{reason}"')
|
|
def step_plan_cancel_reason(context: Context, reason: str) -> None:
|
|
"""Run plan cancel with --reason."""
|
|
context.result = context.runner.invoke(
|
|
plan_app, ["cancel", _PLAN_ULID, "--reason", reason]
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Then steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@then("the plan spec use should succeed")
|
|
def step_plan_use_ok(context: Context) -> None:
|
|
"""Verify plan use succeeded."""
|
|
assert context.result.exit_code == 0, (
|
|
f"Plan use failed ({context.result.exit_code}): {context.result.output}"
|
|
)
|
|
|
|
|
|
@then('the plan spec use should link projects "{proj_a}" and "{proj_b}"')
|
|
def step_plan_use_projects(context: Context, proj_a: str, proj_b: str) -> None:
|
|
"""Verify the service received both project links."""
|
|
call_kwargs = context.mock_service.use_action.call_args
|
|
project_links = call_kwargs[1].get(
|
|
"project_links", call_kwargs[1].get("project_links", [])
|
|
)
|
|
linked_names = [link.project_name for link in project_links]
|
|
assert proj_a in linked_names, f"Expected {proj_a} in {linked_names}"
|
|
assert proj_b in linked_names, f"Expected {proj_b} in {linked_names}"
|
|
|
|
|
|
@then('the plan spec output should contain "{text}"')
|
|
def step_plan_output_contains(context: Context, text: str) -> None:
|
|
"""Verify the output contains expected text (case-insensitive)."""
|
|
output_lower = context.result.output.lower()
|
|
text_lower = text.lower()
|
|
assert text_lower in output_lower, (
|
|
f"Expected '{text}' in output but got:\n{context.result.output}"
|
|
)
|
|
|
|
|
|
@then("the plan spec use should pass invariants to service")
|
|
def step_plan_use_check_invariants(context: Context) -> None:
|
|
"""Verify invariants were passed to use_action."""
|
|
call_kwargs = context.mock_service.use_action.call_args[1]
|
|
invariants = call_kwargs.get("invariants")
|
|
assert invariants is not None, "Expected invariants to be passed"
|
|
assert len(invariants) == 2, f"Expected 2 invariants, got {len(invariants)}"
|
|
texts = [inv.text for inv in invariants]
|
|
assert "No warnings" in texts
|
|
assert "Keep compat" in texts
|
|
|
|
|
|
@then('the plan spec use should pass argument "{name}" with value {value}')
|
|
def step_plan_use_arg_check(context: Context, name: str, value: str) -> None:
|
|
"""Verify argument was passed to use_action."""
|
|
call_kwargs = context.mock_service.use_action.call_args[1]
|
|
arguments = call_kwargs.get("arguments", {})
|
|
assert name in arguments, f"Expected argument {name} in {arguments}"
|
|
assert arguments[name] == int(value), (
|
|
f"Expected {name}={value}, got {arguments[name]}"
|
|
)
|
|
|
|
|
|
@then("the plan spec list should succeed")
|
|
def step_plan_list_ok(context: Context) -> None:
|
|
"""Verify lifecycle-list succeeded."""
|
|
assert context.result.exit_code == 0, (
|
|
f"List failed ({context.result.exit_code}): {context.result.output}"
|
|
)
|
|
|
|
|
|
@then("the plan spec status should succeed")
|
|
def step_plan_status_ok(context: Context) -> None:
|
|
"""Verify plan status succeeded."""
|
|
assert context.result.exit_code == 0, (
|
|
f"Status failed ({context.result.exit_code}): {context.result.output}"
|
|
)
|
|
|
|
|
|
@then('the plan spec status should contain "{text}"')
|
|
def step_plan_status_contains(context: Context, text: str) -> None:
|
|
"""Verify status output contains the expected text."""
|
|
output_lower = context.result.output.lower()
|
|
text_lower = text.lower()
|
|
assert text_lower in output_lower, (
|
|
f"Expected '{text}' in status output but got:\n{context.result.output}"
|
|
)
|
|
|
|
|
|
@then("the plan spec cancel should succeed")
|
|
def step_plan_cancel_ok(context: Context) -> None:
|
|
"""Verify plan cancel succeeded."""
|
|
assert context.result.exit_code == 0, (
|
|
f"Cancel failed ({context.result.exit_code}): {context.result.output}"
|
|
)
|
|
|
|
|
|
@then('the plan spec cancel output should contain "{text}"')
|
|
def step_plan_cancel_contains(context: Context, text: str) -> None:
|
|
"""Verify cancel output contains expected text."""
|
|
assert text in context.result.output, (
|
|
f"Expected '{text}' in cancel output but got:\n{context.result.output}"
|
|
)
|