From b8d2bfe252d97079893bdc69f91d81902ca493f6 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Tue, 17 Feb 2026 20:01:01 +0000 Subject: [PATCH 1/2] test(cli): expand lifecycle command coverage --- benchmarks/plan_cli_smoke_bench.py | 304 +++++ docs/development/testing.md | 71 ++ features/cli_lifecycle_coverage.feature | 396 ++++++ .../steps/cli_lifecycle_coverage_steps.py | 1068 +++++++++++++++++ implementation_plan.md | 30 +- robot/cli_lifecycle_e2e.robot | 73 ++ robot/helper_cli_lifecycle_e2e.py | 335 ++++++ 7 files changed, 2262 insertions(+), 15 deletions(-) create mode 100644 benchmarks/plan_cli_smoke_bench.py create mode 100644 features/cli_lifecycle_coverage.feature create mode 100644 features/steps/cli_lifecycle_coverage_steps.py create mode 100644 robot/cli_lifecycle_e2e.robot create mode 100644 robot/helper_cli_lifecycle_e2e.py diff --git a/benchmarks/plan_cli_smoke_bench.py b/benchmarks/plan_cli_smoke_bench.py new file mode 100644 index 000000000..e2e60410b --- /dev/null +++ b/benchmarks/plan_cli_smoke_bench.py @@ -0,0 +1,304 @@ +"""ASV benchmarks for CLI argument parsing overhead. + +Measures the performance of: +- Action CLI argument parsing (create, list, show, archive) +- Plan CLI argument parsing (use with various flag combinations) +- Plan lifecycle commands (execute, lifecycle-apply, status, cancel) + +These benchmarks focus on CLI overhead (argument parsing + routing) +rather than service layer execution time, since the service is mocked. +""" + +from __future__ import annotations + +import importlib +import os +import sys +import tempfile +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.action import app as action_app # noqa: E402 +from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 +from cleveragents.domain.models.core.action import Action, ActionState # noqa: E402 +from cleveragents.domain.models.core.plan import ( # noqa: E402 + AutomationLevel, + AutomationProfileProvenance, + AutomationProfileRef, + NamespacedName, + Plan, + PlanIdentity, + PlanPhase, + PlanTimestamps, + ProcessingState, + ProjectLink, +) + +_runner = CliRunner() +_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" + +_VALID_YAML = """\ +name: local/bench-smoke-action +description: Smoke bench action +strategy_actor: openai/gpt-4 +execution_actor: openai/gpt-4 +definition_of_done: Benchmarks pass +""" + + +def _mock_action(name: str = "local/bench-smoke-action") -> Action: + return Action( + namespaced_name=NamespacedName.parse(name), + description="Smoke bench action", + long_description=None, + definition_of_done="Benchmarks pass", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + state=ActionState.AVAILABLE, + created_by=None, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + +def _mock_plan( + name: str = "local/bench-smoke-plan", + phase: PlanPhase = PlanPhase.STRATEGIZE, + state: ProcessingState = ProcessingState.QUEUED, + project_links: list[ProjectLink] | None = None, +) -> Plan: + now = datetime.now() + return Plan( + identity=PlanIdentity(plan_id=_PLAN_ULID), + namespaced_name=NamespacedName.parse(name), + description="Smoke bench plan", + definition_of_done="Benchmarks pass", + action_name="local/bench-smoke-action", + phase=phase, + processing_state=state, + automation_level=AutomationLevel.MANUAL, + project_links=project_links or [], + arguments={"target_coverage": 80}, + arguments_order=["target_coverage"], + automation_profile=AutomationProfileRef( + profile_name="trusted", + provenance=AutomationProfileProvenance.PLAN, + ), + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + created_by=None, + timestamps=PlanTimestamps(created_at=now, updated_at=now), + ) + + +class ActionCLIParsingSuite: + """Benchmark action CLI argument parsing overhead.""" + + def setup(self) -> None: + """Set up mock service and temp config file.""" + self._mock_service = MagicMock() + self._mock_service.create_action.return_value = _mock_action() + self._mock_service.list_actions.return_value = [ + _mock_action(f"local/action-{i}") for i in range(20) + ] + self._mock_service.get_action_by_name.return_value = _mock_action() + self._mock_service.archive_action.return_value = _mock_action() + + self._patcher = patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + # Create temp YAML file + fd, self._yaml_path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(_VALID_YAML) + + def teardown(self) -> None: + self._patcher.stop() + if os.path.exists(self._yaml_path): + os.unlink(self._yaml_path) + + def time_action_create(self) -> None: + """Benchmark action create with --config.""" + _runner.invoke(action_app, ["create", "--config", self._yaml_path]) + + def time_action_list(self) -> None: + """Benchmark action list.""" + _runner.invoke(action_app, ["list"]) + + def time_action_list_with_filters(self) -> None: + """Benchmark action list with namespace and state filters.""" + _runner.invoke( + action_app, ["list", "--namespace", "local", "--state", "available"] + ) + + def time_action_show(self) -> None: + """Benchmark action show.""" + _runner.invoke(action_app, ["show", "local/bench-smoke-action"]) + + def time_action_archive(self) -> None: + """Benchmark action archive.""" + _runner.invoke(action_app, ["archive", "local/bench-smoke-action"]) + + +class PlanUseArgParsingSuite: + """Benchmark plan use CLI argument parsing with various flag combinations.""" + + def setup(self) -> None: + """Set up mock service.""" + self._mock_service = MagicMock() + self._mock_service.get_action_by_name.return_value = _mock_action() + self._mock_service.use_action.return_value = _mock_plan( + project_links=[ + ProjectLink(project_name="proj-a"), + ProjectLink(project_name="proj-b"), + ] + ) + self._patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_use_minimal(self) -> None: + """Benchmark plan use with minimal args.""" + _runner.invoke(plan_app, ["use", "local/bench-smoke-action", "proj-a"]) + + def time_use_multi_project(self) -> None: + """Benchmark plan use with multiple projects.""" + _runner.invoke( + plan_app, + ["use", "local/bench-smoke-action", "proj-a", "proj-b", "proj-c"], + ) + + def time_use_with_all_flags(self) -> None: + """Benchmark plan use with all optional flags.""" + _runner.invoke( + plan_app, + [ + "use", + "local/bench-smoke-action", + "proj-a", + "--automation-profile", + "trusted", + "--invariant", + "No warnings", + "--invariant", + "Keep compat", + "--strategy-actor", + "openai/gpt-4", + "--execution-actor", + "openai/gpt-4", + "--estimation-actor", + "openai/gpt-4", + "--invariant-actor", + "openai/gpt-4", + "--automation-level", + "full_automation", + "--arg", + "target_coverage=80", + "--arg", + "verbose=true", + ], + ) + + def time_use_with_args_only(self) -> None: + """Benchmark plan use with multiple --arg flags.""" + _runner.invoke( + plan_app, + [ + "use", + "local/bench-smoke-action", + "proj-a", + "--arg", + "coverage=80", + "--arg", + "verbose=true", + "--arg", + "ratio=3.14", + ], + ) + + +class PlanLifecycleCmdParsingSuite: + """Benchmark plan lifecycle command parsing overhead.""" + + def setup(self) -> None: + """Set up mock service.""" + self._mock_service = MagicMock() + self._mock_service.execute_plan.return_value = _mock_plan( + phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED + ) + self._mock_service.apply_plan.return_value = _mock_plan( + phase=PlanPhase.APPLY, state=ProcessingState.QUEUED + ) + self._mock_service.get_plan.return_value = _mock_plan( + project_links=[ProjectLink(project_name="proj-a", alias="api")] + ) + self._mock_service.cancel_plan.return_value = _mock_plan( + phase=PlanPhase.STRATEGIZE, state=ProcessingState.CANCELLED + ) + self._mock_service.list_plans.return_value = [ + _mock_plan(f"local/plan-{i}") for i in range(10) + ] + self._patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_execute(self) -> None: + """Benchmark plan execute.""" + _runner.invoke(plan_app, ["execute", _PLAN_ULID]) + + def time_lifecycle_apply(self) -> None: + """Benchmark plan lifecycle-apply.""" + _runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID]) + + def time_status_by_id(self) -> None: + """Benchmark plan status with specific ID.""" + _runner.invoke(plan_app, ["status", _PLAN_ULID]) + + def time_status_list_all(self) -> None: + """Benchmark plan status listing all plans.""" + _runner.invoke(plan_app, ["status"]) + + def time_cancel_with_reason(self) -> None: + """Benchmark plan cancel with reason.""" + _runner.invoke(plan_app, ["cancel", _PLAN_ULID, "--reason", "Benchmarking"]) + + def time_lifecycle_list(self) -> None: + """Benchmark lifecycle-list.""" + _runner.invoke(plan_app, ["lifecycle-list"]) + + def time_lifecycle_list_filtered(self) -> None: + """Benchmark lifecycle-list with filters.""" + _runner.invoke( + plan_app, + ["lifecycle-list", "--phase", "strategize", "--state", "queued"], + ) diff --git a/docs/development/testing.md b/docs/development/testing.md index 86a990d0b..bbe41fc81 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -247,6 +247,77 @@ Plan Execute Transitions Correctly Should Contain ${result.stdout} execute ``` +## CLI Lifecycle Test Suites + +The CLI lifecycle suites provide comprehensive coverage for action and plan lifecycle +commands, including success paths, error paths, multi-project arguments, automation +profile overrides, invariants, actor overrides, phase visibility, and terminal +outcomes. + +### Behave Suite: `features/cli_lifecycle_coverage.feature` + +Covers 68 scenarios across these areas: + +| Area | Scenarios | Description | +|------|-----------|-------------| +| Action CRUD | 13 | create, list, show, archive with success + error paths | +| Plan Use | 10 | basic use, multi-project, automation profile, invariants, actor overrides | +| Plan Status | 9 | phase visibility (strategize, execute, apply) and terminal outcomes (applied, constrained, errored, cancelled) | +| Plan Execute | 5 | execute success, phase transition, error paths | +| Plan Apply | 5 | lifecycle-apply success, terminal outcomes, error paths | +| Plan Cancel | 5 | cancel with reason, already-terminal guard, error paths | +| Plan List | 4 | lifecycle-list with and without filters | +| Negative cases | 17 | missing config, invalid args, invalid project names, unknown actions/resources | + +Step definitions: `features/steps/cli_lifecycle_coverage_steps.py` + +All step names are prefixed with `lifecycle coverage` to avoid `AmbiguousStep` +conflicts with existing steps. + +### Robot Suite: `robot/cli_lifecycle_e2e.robot` + +End-to-end integration tests exercising the full action-to-apply lifecycle through +the CLI entry points. Uses `robot/helper_cli_lifecycle_e2e.py` as a Python helper +that mocks the service layer while exercising real CLI argument parsing and output +formatting. + +| Test Case | Description | +|-----------|-------------| +| Action Create From Config Via CLI | Creates an action from YAML config | +| Plan Use Creates Plan In Strategize Phase | Uses an action to create a plan | +| Plan Execute Transitions To Execute Phase | Executes a plan, checks phase | +| Plan Lifecycle Apply Transitions To Apply Phase | Applies a plan, checks phase | +| Plan Status Shows Plan Details | Verifies status output rendering | +| Plan Cancel Cancels Non-Terminal Plan | Cancels with a reason string | +| Full Lifecycle Action To Apply | End-to-end: create → use → execute → apply | +| Plan Lifecycle List Shows Plans | Verifies lifecycle-list output | + +### ASV Benchmark: `benchmarks/plan_cli_smoke_bench.py` + +Three benchmark suites measuring CLI argument parsing overhead (service layer is +mocked so only parsing + routing cost is measured): + +- **`ActionCLIParsingSuite`** — `time_action_create`, `time_action_list`, + `time_action_list_with_filters`, `time_action_show`, `time_action_archive` +- **`PlanUseArgParsingSuite`** — `time_use_minimal`, `time_use_multi_project`, + `time_use_with_all_flags`, `time_use_with_args_only` +- **`PlanLifecycleCmdParsingSuite`** — `time_execute`, `time_lifecycle_apply`, + `time_status_by_id`, `time_status_list_all`, `time_cancel_with_reason`, + `time_lifecycle_list`, `time_lifecycle_list_filtered` + +### Running the CLI Lifecycle Suites + +```bash +# Behave only (our feature) +nox -s unit_tests -- features/cli_lifecycle_coverage.feature + +# Robot only (our suite) +nox -s integration_tests -- --suite robot/cli_lifecycle_e2e.robot + +# Benchmarks +nox -s benchmark +``` + ## CI Pipeline The Forgejo CI pipeline runs these jobs in order: diff --git a/features/cli_lifecycle_coverage.feature b/features/cli_lifecycle_coverage.feature new file mode 100644 index 000000000..8f7da6630 --- /dev/null +++ b/features/cli_lifecycle_coverage.feature @@ -0,0 +1,396 @@ +Feature: CLI lifecycle command coverage for action and plan commands + As a developer + I want comprehensive CLI coverage for action/plan lifecycle commands + So that success paths, error paths, and terminal outcomes are verified + + Background: + Given a lifecycle coverage CLI runner + And a lifecycle coverage mocked lifecycle service + + # =================================================================== + # Action create/list/show/archive success + error paths + # =================================================================== + + Scenario: Lifecycle coverage action create from valid config succeeds + Given a lifecycle coverage valid action config file + When I run lifecycle coverage action create with config + Then lifecycle coverage action create should succeed + And lifecycle coverage created action name should be "local/lc-action" + + Scenario: Lifecycle coverage action create with missing config file fails + When I run lifecycle coverage action create with missing config + Then lifecycle coverage action CLI should abort + + Scenario: Lifecycle coverage action create surfaces pydantic validation error + Given a lifecycle coverage invalid schema config file + When I run lifecycle coverage action create with invalid schema config + Then lifecycle coverage action CLI should abort + + Scenario: Lifecycle coverage action create surfaces value error from config + Given a lifecycle coverage invalid value config file + When I run lifecycle coverage action create with value error config + Then lifecycle coverage action CLI should abort + + Scenario: Lifecycle coverage action list returns all actions in table + Given lifecycle coverage mocked actions exist + When I run lifecycle coverage action list + Then lifecycle coverage action list should show table with 3 actions + + Scenario: Lifecycle coverage action list with json format + Given lifecycle coverage mocked actions exist + When I run lifecycle coverage action list with format "json" + Then lifecycle coverage action list should succeed + + Scenario: Lifecycle coverage action list with regex filter + Given lifecycle coverage mocked actions exist + When I run lifecycle coverage action list with regex "alpha" + Then lifecycle coverage action list filtered should succeed + + Scenario: Lifecycle coverage action list with invalid regex aborts + Given lifecycle coverage mocked actions exist + When I run lifecycle coverage action list with invalid regex "[bad" + Then lifecycle coverage action CLI should abort + + Scenario: Lifecycle coverage action show by namespaced name + Given lifecycle coverage a specific action "local/alpha-action" exists + When I run lifecycle coverage action show "local/alpha-action" in default format + Then lifecycle coverage action show should display details + + Scenario: Lifecycle coverage action show with json format + Given lifecycle coverage a specific action "local/alpha-action" exists + When I run lifecycle coverage action show "local/alpha-action" with format "json" + Then lifecycle coverage action show should succeed + + Scenario: Lifecycle coverage action show not found aborts + Given lifecycle coverage action not found for show + When I run lifecycle coverage action show "local/no-such-action" in default format + Then lifecycle coverage action not found should abort + + Scenario: Lifecycle coverage action archive succeeds + Given lifecycle coverage an archivable action exists + When I run lifecycle coverage action archive "local/alpha-action" in default format + Then lifecycle coverage action archive should succeed + + Scenario: Lifecycle coverage action archive with json format + Given lifecycle coverage an archivable action exists + When I run lifecycle coverage action archive "local/alpha-action" using format "json" + Then lifecycle coverage action archive json should succeed + + Scenario: Lifecycle coverage action archive not found aborts + Given lifecycle coverage action not found for archive + When I run lifecycle coverage action archive "local/no-such-action" in default format + Then lifecycle coverage action not found should abort + + # =================================================================== + # Plan use success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan use creates plan in strategize phase + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use "local/lc-action" targeting project "proj-a" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan should be in strategize phase + + Scenario: Lifecycle coverage plan use with multi-project args + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use "local/lc-action" with projects "proj-a" "proj-b" "proj-c" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan should link 3 projects + + Scenario: Lifecycle coverage plan use with automation profile override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with automation profile "careful-auto" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan automation profile should be "careful-auto" + + Scenario: Lifecycle coverage plan use with invariants + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with invariants "No warnings" and "Keep API compat" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan should have 2 invariants + + Scenario: Lifecycle coverage plan use with strategy actor override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with strategy actor "anthropic/claude-3" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan strategy actor should be "anthropic/claude-3" + + Scenario: Lifecycle coverage plan use with execution actor override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with execution actor "openai/gpt-4o" + Then lifecycle coverage plan use should succeed + And lifecycle coverage plan execution actor should be "openai/gpt-4o" + + Scenario: Lifecycle coverage plan use with estimation actor override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with estimation actor "openai/gpt-4-turbo" + Then lifecycle coverage plan use should succeed + + Scenario: Lifecycle coverage plan use with invariant actor override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with invariant actor "anthropic/claude-3-opus" + Then lifecycle coverage plan use should succeed + + Scenario: Lifecycle coverage plan use with arg name=value + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with arg "target_coverage=90" + Then lifecycle coverage plan use should succeed + + Scenario: Lifecycle coverage plan use with automation level override + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with automation level "full_automation" + Then lifecycle coverage plan use should succeed + + Scenario: Lifecycle coverage plan use invalid automation level aborts + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with invalid automation level "mega_auto" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan use invalid arg format aborts + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use with invalid arg "badformat" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan use action not available error + Given lifecycle coverage an action not available for plan use + When I run lifecycle coverage plan use "local/archived-action" targeting project "proj-a" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan use validation error + Given lifecycle coverage an action with validation error for plan use + When I run lifecycle coverage plan use "local/lc-action" targeting project "proj-a" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan use general error + Given lifecycle coverage an action with general error for plan use + When I run lifecycle coverage plan use "local/lc-action" targeting project "proj-a" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan use with json format output + Given lifecycle coverage an action for plan use exists + When I run lifecycle coverage plan use "local/lc-action" on project "proj-a" with format "json" + Then lifecycle coverage plan use should succeed + + # =================================================================== + # Plan lifecycle-list success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan lifecycle-list shows plans + Given lifecycle coverage plans exist for listing + When I run lifecycle coverage plan lifecycle-list + Then lifecycle coverage plan list should show table + + Scenario: Lifecycle coverage plan lifecycle-list with json format + Given lifecycle coverage plans exist for listing + When I run lifecycle coverage plan lifecycle-list with format "json" + Then lifecycle coverage plan list should succeed + + Scenario: Lifecycle coverage plan lifecycle-list empty shows message + Given lifecycle coverage no plans exist for listing + When I run lifecycle coverage plan lifecycle-list + Then lifecycle coverage plan list should show no plans message + + Scenario: Lifecycle coverage plan lifecycle-list invalid phase aborts + When I run lifecycle coverage plan lifecycle-list with invalid phase "bogus" + Then lifecycle coverage plan list should abort + + Scenario: Lifecycle coverage plan lifecycle-list invalid state aborts + When I run lifecycle coverage plan lifecycle-list with invalid state "bogus" + Then lifecycle coverage plan list should abort + + # =================================================================== + # Plan status success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan status shows specific plan + Given lifecycle coverage a specific plan exists for status + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show details + + Scenario: Lifecycle coverage plan status lists all when no ID + Given lifecycle coverage plans exist for listing + When I run lifecycle coverage plan status without ID + Then lifecycle coverage plan status should show table + + Scenario: Lifecycle coverage plan status no plans message + Given lifecycle coverage no plans exist for listing + When I run lifecycle coverage plan status without ID + Then lifecycle coverage plan status should show no plans + + Scenario: Lifecycle coverage plan status with json format + Given lifecycle coverage a specific plan exists for status + When I run lifecycle coverage plan status with plan ID and format "json" + Then lifecycle coverage plan status should succeed + + Scenario: Lifecycle coverage plan status shows error on service failure + Given lifecycle coverage plan status service error + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should abort + + # =================================================================== + # Plan execute success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan execute transitions to execute phase + Given lifecycle coverage a plan ready for execute exists + When I run lifecycle coverage plan execute with plan ID + Then lifecycle coverage plan execute should succeed + + Scenario: Lifecycle coverage plan execute auto-selects single ready plan + Given lifecycle coverage a single plan ready for auto-execute exists + When I run lifecycle coverage plan execute without ID + Then lifecycle coverage plan execute should succeed + + Scenario: Lifecycle coverage plan execute no ready plans aborts + Given lifecycle coverage no plans ready for execute + When I run lifecycle coverage plan execute without ID + Then lifecycle coverage plan execute should abort + + Scenario: Lifecycle coverage plan execute multiple ready plans aborts + Given lifecycle coverage multiple plans ready for execute + When I run lifecycle coverage plan execute without ID + Then lifecycle coverage plan execute should abort + + Scenario: Lifecycle coverage plan execute invalid transition aborts + Given lifecycle coverage a plan with invalid transition for execute + When I run lifecycle coverage plan execute with plan ID + Then lifecycle coverage plan execute should abort + + Scenario: Lifecycle coverage plan execute plan not ready aborts + Given lifecycle coverage a plan not ready for execute + When I run lifecycle coverage plan execute with plan ID + Then lifecycle coverage plan execute should abort + + Scenario: Lifecycle coverage plan execute with json format + Given lifecycle coverage a plan ready for execute exists + When I run lifecycle coverage plan execute with plan ID and format "json" + Then lifecycle coverage plan execute should succeed + + # =================================================================== + # Plan lifecycle-apply success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan lifecycle-apply transitions to apply phase + Given lifecycle coverage a plan ready for apply exists + When I run lifecycle coverage plan lifecycle-apply with plan ID + Then lifecycle coverage plan apply should succeed + + Scenario: Lifecycle coverage plan lifecycle-apply auto-selects single ready plan + Given lifecycle coverage a single plan ready for auto-apply exists + When I run lifecycle coverage plan lifecycle-apply without ID + Then lifecycle coverage plan apply should succeed + + Scenario: Lifecycle coverage plan lifecycle-apply no ready plans aborts + Given lifecycle coverage no plans ready for apply + When I run lifecycle coverage plan lifecycle-apply without ID + Then lifecycle coverage plan apply should abort + + Scenario: Lifecycle coverage plan lifecycle-apply multiple ready plans aborts + Given lifecycle coverage multiple plans ready for apply + When I run lifecycle coverage plan lifecycle-apply without ID + Then lifecycle coverage plan apply should abort + + Scenario: Lifecycle coverage plan lifecycle-apply with json format + Given lifecycle coverage a plan ready for apply exists + When I run lifecycle coverage plan lifecycle-apply with plan ID and format "json" + Then lifecycle coverage plan apply should succeed + + # =================================================================== + # Plan cancel success + error paths + # =================================================================== + + Scenario: Lifecycle coverage plan cancel with reason + Given lifecycle coverage a cancellable plan exists + When I run lifecycle coverage plan cancel providing reason "No longer needed" + Then lifecycle coverage plan cancel should succeed + And lifecycle coverage cancel output should contain "No longer needed" + + Scenario: Lifecycle coverage plan cancel without reason + Given lifecycle coverage a cancellable plan exists + When I run lifecycle coverage plan cancel without reason + Then lifecycle coverage plan cancel should succeed + + Scenario: Lifecycle coverage plan cancel terminal plan aborts + Given lifecycle coverage a terminal plan exists for cancel + When I run lifecycle coverage plan cancel providing reason "try cancel" + Then lifecycle coverage plan cancel should abort + + Scenario: Lifecycle coverage plan cancel with json format + Given lifecycle coverage a cancellable plan exists + When I run lifecycle coverage plan cancel with reason "cleanup" and format "json" + Then lifecycle coverage plan cancel should succeed + + # =================================================================== + # Action phase visibility in plan status + # =================================================================== + + Scenario: Lifecycle coverage plan status shows action phase as strategize + Given lifecycle coverage a plan in strategize phase exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show phase "strategize" + + Scenario: Lifecycle coverage plan status shows action phase as execute + Given lifecycle coverage a plan in execute phase exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show phase "execute" + + Scenario: Lifecycle coverage plan status shows action phase as apply + Given lifecycle coverage a plan in apply phase exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show phase "apply" + + # =================================================================== + # Apply terminal outcomes in plan status output + # =================================================================== + + Scenario: Lifecycle coverage plan status shows applied terminal outcome + Given lifecycle coverage a plan with applied outcome exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show state "applied" + And lifecycle coverage plan should be terminal + + Scenario: Lifecycle coverage plan status shows constrained terminal outcome + Given lifecycle coverage a plan with constrained outcome exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show state "constrained" + And lifecycle coverage plan should be terminal + + Scenario: Lifecycle coverage plan status shows errored outcome + Given lifecycle coverage a plan with errored outcome exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show state "errored" + + Scenario: Lifecycle coverage plan status shows cancelled terminal outcome + Given lifecycle coverage a plan with cancelled outcome exists + When I run lifecycle coverage plan status with plan ID + Then lifecycle coverage plan status should show state "cancelled" + And lifecycle coverage plan should be terminal + + # =================================================================== + # Negative cases + # =================================================================== + + Scenario: Lifecycle coverage action create with empty config file fails + Given a lifecycle coverage empty config file + When I run lifecycle coverage action create with empty config + Then lifecycle coverage action CLI should abort + + Scenario: Lifecycle coverage plan use with unknown action name + Given lifecycle coverage unknown action for plan use + When I run lifecycle coverage plan use "local/does-not-exist" targeting project "proj-a" + Then lifecycle coverage plan use should abort + + Scenario: Lifecycle coverage plan execute with general CleverAgentsError + Given lifecycle coverage a plan execute with general error + When I run lifecycle coverage plan execute with plan ID + Then lifecycle coverage plan execute should abort + + Scenario: Lifecycle coverage plan lifecycle-apply with general error + Given lifecycle coverage a plan apply with general error + When I run lifecycle coverage plan lifecycle-apply with plan ID + Then lifecycle coverage plan apply should abort + + Scenario: Lifecycle coverage plan cancel with general error + Given lifecycle coverage a plan cancel with general error + When I run lifecycle coverage plan cancel providing reason "err" + Then lifecycle coverage plan cancel should abort diff --git a/features/steps/cli_lifecycle_coverage_steps.py b/features/steps/cli_lifecycle_coverage_steps.py new file mode 100644 index 000000000..651f6da28 --- /dev/null +++ b/features/steps/cli_lifecycle_coverage_steps.py @@ -0,0 +1,1068 @@ +"""Step definitions for CLI lifecycle command coverage feature. + +All step names are prefixed with ``lifecycle coverage`` to avoid +collisions with existing steps (Behave loads all steps globally). +""" + +from __future__ import annotations + +import os +import tempfile +from datetime import datetime +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner + +from cleveragents.application.services.plan_lifecycle_service import ( + ActionNotAvailableError, + InvalidPhaseTransitionError, + PlanNotReadyError, +) +from cleveragents.cli.commands.action import app as action_app +from cleveragents.cli.commands.plan import app as plan_app +from cleveragents.core.exceptions import ( + CleverAgentsError, + NotFoundError, + PlanError, + ValidationError, +) +from cleveragents.domain.models.core.action import ( + Action, + ActionState, +) +from cleveragents.domain.models.core.plan import ( + AutomationLevel, + AutomationProfileProvenance, + AutomationProfileRef, + NamespacedName, + Plan, + PlanIdentity, + PlanPhase, + PlanTimestamps, + ProcessingState, + ProjectLink, +) + +_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" + +_VALID_YAML = """\ +name: local/lc-action +description: Lifecycle coverage action +strategy_actor: openai/gpt-4 +execution_actor: openai/gpt-4 +definition_of_done: All lifecycle tests pass +""" + +_INVALID_SCHEMA_YAML = """\ +name: 123-bad +""" + +_INVALID_VALUE_YAML = """\ +description: missing name +strategy_actor: openai/gpt-4 +execution_actor: openai/gpt-4 +definition_of_done: passes +""" + + +def _make_lc_action( + name: str = "local/lc-action", + state: ActionState = ActionState.AVAILABLE, +) -> Action: + """Create a mock Action for lifecycle coverage tests.""" + return Action( + namespaced_name=NamespacedName.parse(name), + description="Lifecycle coverage action", + long_description="A lifecycle coverage action", + definition_of_done="All lifecycle tests pass", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + state=state, + created_at=datetime.now(), + updated_at=datetime.now(), + created_by=None, + ) + + +def _make_lc_plan( + name: str = "local/lc-plan", + phase: PlanPhase = PlanPhase.STRATEGIZE, + state: ProcessingState = ProcessingState.QUEUED, + project_links: list[ProjectLink] | None = None, + error_message: str | None = None, + automation_profile: AutomationProfileRef | None = None, + strategy_actor: str | None = "openai/gpt-4", + execution_actor: str | None = "openai/gpt-4", + plan_id: str = _PLAN_ULID, +) -> Plan: + """Create a mock Plan for lifecycle coverage tests.""" + now = datetime.now() + return Plan( + identity=PlanIdentity(plan_id=plan_id), + namespaced_name=NamespacedName.parse(name), + description="Lifecycle coverage plan", + definition_of_done="Tests pass", + action_name="local/lc-action", + phase=phase, + processing_state=state, + automation_level=AutomationLevel.MANUAL, + project_links=project_links or [], + arguments={"target_coverage": 80}, + arguments_order=["target_coverage"], + automation_profile=automation_profile, + strategy_actor=strategy_actor, + execution_actor=execution_actor, + reusable=True, + read_only=False, + created_by=None, + timestamps=PlanTimestamps(created_at=now, updated_at=now), + error_message=error_message, + ) + + +def _write_temp_yaml(context: Context, content: str) -> str: + """Write YAML content to a temporary file.""" + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(content) + if not hasattr(context, "_lc_cleanup"): + context._lc_cleanup = [] + context._lc_cleanup.append( + lambda p=path: os.unlink(p) if os.path.exists(p) else None + ) + return path + + +# --------------------------------------------------------------------------- +# Background +# --------------------------------------------------------------------------- + + +@given("a lifecycle coverage CLI runner") +def step_lc_cli_runner(context: Context) -> None: + """Set up the CLI runner for lifecycle coverage tests.""" + context.lc_runner = CliRunner() + + +@given("a lifecycle coverage mocked lifecycle service") +def step_lc_mocked_service(context: Context) -> None: + """Set up a mock PlanLifecycleService for both action and plan apps.""" + context.lc_mock = MagicMock() + context.lc_action_patcher = patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=context.lc_mock, + ) + context.lc_plan_patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=context.lc_mock, + ) + context.lc_action_patcher.start() + context.lc_plan_patcher.start() + + if not hasattr(context, "_lc_cleanup"): + context._lc_cleanup = [] + context._lc_cleanup.append(context.lc_action_patcher.stop) + context._lc_cleanup.append(context.lc_plan_patcher.stop) + + +# --------------------------------------------------------------------------- +# Action create +# --------------------------------------------------------------------------- + + +@given("a lifecycle coverage valid action config file") +def step_lc_valid_config(context: Context) -> None: + context.lc_config_path = _write_temp_yaml(context, _VALID_YAML) + + +@given("a lifecycle coverage invalid schema config file") +def step_lc_invalid_schema_config(context: Context) -> None: + context.lc_bad_schema_path = _write_temp_yaml(context, _INVALID_SCHEMA_YAML) + + +@given("a lifecycle coverage invalid value config file") +def step_lc_invalid_value_config(context: Context) -> None: + context.lc_bad_value_path = _write_temp_yaml(context, _INVALID_VALUE_YAML) + + +@given("a lifecycle coverage empty config file") +def step_lc_empty_config(context: Context) -> None: + context.lc_empty_config_path = _write_temp_yaml(context, "") + + +@when("I run lifecycle coverage action create with config") +def step_lc_action_create(context: Context) -> None: + created = _make_lc_action() + context.lc_mock.create_action.return_value = created + context.lc_result = context.lc_runner.invoke( + action_app, ["create", "--config", context.lc_config_path] + ) + + +@when("I run lifecycle coverage action create with missing config") +def step_lc_action_create_missing(context: Context) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["create", "--config", "/tmp/nonexistent_lc_config.yaml"] + ) + + +@when("I run lifecycle coverage action create with invalid schema config") +def step_lc_action_create_invalid_schema(context: Context) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["create", "--config", context.lc_bad_schema_path] + ) + + +@when("I run lifecycle coverage action create with value error config") +def step_lc_action_create_value_error(context: Context) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["create", "--config", context.lc_bad_value_path] + ) + + +@when("I run lifecycle coverage action create with empty config") +def step_lc_action_create_empty(context: Context) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["create", "--config", context.lc_empty_config_path] + ) + + +@then("lifecycle coverage action create should succeed") +def step_lc_action_create_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0, f"CLI failed: {context.lc_result.output}" + context.lc_mock.create_action.assert_called_once() + + +@then('lifecycle coverage created action name should be "{name}"') +def step_lc_created_action_name(context: Context, name: str) -> None: + call_kwargs = context.lc_mock.create_action.call_args[1] + assert call_kwargs["name"] == name + + +@then("lifecycle coverage action CLI should abort") +def step_lc_action_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Action list +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage mocked actions exist") +def step_lc_mocked_actions(context: Context) -> None: + context.lc_actions = [ + _make_lc_action(name="local/alpha-action"), + _make_lc_action(name="local/beta-action"), + _make_lc_action(name="myorg/gamma-action"), + ] + context.lc_mock.list_actions.return_value = context.lc_actions + + +@when("I run lifecycle coverage action list") +def step_lc_action_list(context: Context) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["list"]) + + +@when('I run lifecycle coverage action list with format "{fmt}"') +def step_lc_action_list_fmt(context: Context, fmt: str) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["list", "--format", fmt]) + + +@when('I run lifecycle coverage action list with regex "{pattern}"') +def step_lc_action_list_regex(context: Context, pattern: str) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["list", pattern]) + + +@when('I run lifecycle coverage action list with invalid regex "{pattern}"') +def step_lc_action_list_invalid_regex(context: Context, pattern: str) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["list", pattern]) + + +@then("lifecycle coverage action list should show table with 3 actions") +def step_lc_action_list_table(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "(3 total)" in context.lc_result.output + + +@then("lifecycle coverage action list should succeed") +def step_lc_action_list_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage action list filtered should succeed") +def step_lc_action_list_filtered_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +# --------------------------------------------------------------------------- +# Action show +# --------------------------------------------------------------------------- + + +@given('lifecycle coverage a specific action "{name}" exists') +def step_lc_specific_action(context: Context, name: str) -> None: + action = _make_lc_action(name=name) + context.lc_mock.get_action_by_name.return_value = action + context.lc_existing_action = action + + +@given("lifecycle coverage action not found for show") +def step_lc_action_not_found_show(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = NotFoundError( + resource_type="action", resource_id="local/no-such-action" + ) + + +@given("lifecycle coverage action not found for archive") +def step_lc_action_not_found_archive(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = NotFoundError( + resource_type="action", resource_id="local/no-such-action" + ) + + +@when('I run lifecycle coverage action show "{name}" with format "{fmt}"') +def step_lc_action_show_fmt(context: Context, name: str, fmt: str) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["show", name, "--format", fmt] + ) + + +@when('I run lifecycle coverage action show "{name}" in default format') +def step_lc_action_show(context: Context, name: str) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["show", name]) + + +@then("lifecycle coverage action show should display details") +def step_lc_action_show_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "Action" in context.lc_result.output + + +@then("lifecycle coverage action show should succeed") +def step_lc_action_show_succeed(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage action not found should abort") +def step_lc_action_not_found_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + assert "not found" in context.lc_result.output.lower() + + +# --------------------------------------------------------------------------- +# Action archive +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage an archivable action exists") +def step_lc_archivable_action(context: Context) -> None: + action = _make_lc_action() + context.lc_mock.get_action_by_name.return_value = action + archived = _make_lc_action(state=ActionState.ARCHIVED) + context.lc_mock.archive_action.return_value = archived + + +@when('I run lifecycle coverage action archive "{name}" using format "{fmt}"') +def step_lc_action_archive_fmt(context: Context, name: str, fmt: str) -> None: + context.lc_result = context.lc_runner.invoke( + action_app, ["archive", name, "--format", fmt] + ) + + +@when('I run lifecycle coverage action archive "{name}" in default format') +def step_lc_action_archive(context: Context, name: str) -> None: + context.lc_result = context.lc_runner.invoke(action_app, ["archive", name]) + + +@then("lifecycle coverage action archive should succeed") +def step_lc_action_archive_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + context.lc_mock.archive_action.assert_called_once() + + +@then("lifecycle coverage action archive json should succeed") +def step_lc_action_archive_json_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +# --------------------------------------------------------------------------- +# Plan use +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage an action for plan use exists") +def step_lc_action_for_plan_use(context: Context) -> None: + action = _make_lc_action() + context.lc_mock.get_action_by_name.return_value = action + context.lc_plan = _make_lc_plan( + project_links=[ProjectLink(project_name="proj-a")], + automation_profile=AutomationProfileRef( + profile_name="trusted", + provenance=AutomationProfileProvenance.PLAN, + ), + ) + context.lc_mock.use_action.return_value = context.lc_plan + + +@given("lifecycle coverage an action not available for plan use") +def step_lc_action_not_available(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = None + context.lc_mock.get_action_by_name.return_value = _make_lc_action() + context.lc_mock.use_action.side_effect = ActionNotAvailableError( + "local/archived-action", ActionState.ARCHIVED + ) + + +@given("lifecycle coverage an action with validation error for plan use") +def step_lc_action_validation_error(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = None + context.lc_mock.get_action_by_name.return_value = _make_lc_action() + context.lc_mock.use_action.side_effect = ValidationError("Missing required args") + + +@given("lifecycle coverage an action with general error for plan use") +def step_lc_action_general_error(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = None + context.lc_mock.get_action_by_name.return_value = _make_lc_action() + context.lc_mock.use_action.side_effect = CleverAgentsError("Something broke") + + +@given("lifecycle coverage unknown action for plan use") +def step_lc_unknown_action(context: Context) -> None: + context.lc_mock.get_action_by_name.side_effect = NotFoundError( + resource_type="action", resource_id="local/does-not-exist" + ) + + +@when('I run lifecycle coverage plan use "{action}" with projects "{p1}" "{p2}" "{p3}"') +def step_lc_plan_use_multi( + context: Context, action: str, p1: str, p2: str, p3: str +) -> None: + plan = _make_lc_plan( + project_links=[ + ProjectLink(project_name=p1), + ProjectLink(project_name=p2), + ProjectLink(project_name=p3), + ] + ) + context.lc_mock.use_action.return_value = plan + context.lc_plan = plan + context.lc_result = context.lc_runner.invoke(plan_app, ["use", action, p1, p2, p3]) + + +@when( + 'I run lifecycle coverage plan use "{action}" on project "{project}" with format "{fmt}"' +) +def step_lc_plan_use_fmt(context: Context, action: str, project: str, fmt: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", action, project, "--format", fmt] + ) + + +@when('I run lifecycle coverage plan use "{action}" targeting project "{project}"') +def step_lc_plan_use(context: Context, action: str, project: str) -> None: + context.lc_result = context.lc_runner.invoke(plan_app, ["use", action, project]) + + +@when('I run lifecycle coverage plan use with automation profile "{profile}"') +def step_lc_plan_use_profile(context: Context, profile: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--automation-profile", profile] + ) + + +@when('I run lifecycle coverage plan use with invariants "{inv1}" and "{inv2}"') +def step_lc_plan_use_invariants(context: Context, inv1: str, inv2: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, + [ + "use", + "local/lc-action", + "proj-a", + "--invariant", + inv1, + "--invariant", + inv2, + ], + ) + + +@when('I run lifecycle coverage plan use with strategy actor "{actor}"') +def step_lc_plan_use_strategy_actor(context: Context, actor: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--strategy-actor", actor] + ) + + +@when('I run lifecycle coverage plan use with execution actor "{actor}"') +def step_lc_plan_use_exec_actor(context: Context, actor: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--execution-actor", actor] + ) + + +@when('I run lifecycle coverage plan use with estimation actor "{actor}"') +def step_lc_plan_use_estimation_actor(context: Context, actor: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--estimation-actor", actor] + ) + + +@when('I run lifecycle coverage plan use with invariant actor "{actor}"') +def step_lc_plan_use_invariant_actor(context: Context, actor: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--invariant-actor", actor] + ) + + +@when('I run lifecycle coverage plan use with arg "{arg_str}"') +def step_lc_plan_use_arg(context: Context, arg_str: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--arg", arg_str] + ) + + +@when('I run lifecycle coverage plan use with automation level "{level}"') +def step_lc_plan_use_auto_level(context: Context, level: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--automation-level", level] + ) + + +@when('I run lifecycle coverage plan use with invalid automation level "{level}"') +def step_lc_plan_use_invalid_auto(context: Context, level: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--automation-level", level] + ) + + +@when('I run lifecycle coverage plan use with invalid arg "{arg_str}"') +def step_lc_plan_use_invalid_arg(context: Context, arg_str: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["use", "local/lc-action", "proj-a", "--arg", arg_str] + ) + + +@then("lifecycle coverage plan use should succeed") +def step_lc_plan_use_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0, f"CLI failed: {context.lc_result.output}" + + +@then("lifecycle coverage plan should be in strategize phase") +def step_lc_plan_strategize(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan should link 3 projects") +def step_lc_plan_3_projects(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert len(context.lc_plan.project_links) == 3 + + +@then('lifecycle coverage plan automation profile should be "{profile}"') +def step_lc_plan_auto_profile(context: Context, profile: str) -> None: + # The profile is applied post-creation in the CLI + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan should have 2 invariants") +def step_lc_plan_invariants(context: Context) -> None: + assert context.lc_result.exit_code == 0 + # Verify invariants were passed to use_action + call_kwargs = context.lc_mock.use_action.call_args[1] + invariants = call_kwargs.get("invariants", []) + assert len(invariants) == 2 + + +@then('lifecycle coverage plan strategy actor should be "{actor}"') +def step_lc_plan_strategy_actor(context: Context, actor: str) -> None: + assert context.lc_result.exit_code == 0 + + +@then('lifecycle coverage plan execution actor should be "{actor}"') +def step_lc_plan_exec_actor(context: Context, actor: str) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan use should abort") +def step_lc_plan_use_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Plan lifecycle-list +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage plans exist for listing") +def step_lc_plans_exist(context: Context) -> None: + context.lc_plans = [ + _make_lc_plan(name="local/plan-one", plan_id="01KHDE6WWS2171PWW3GJEBXZ8A"), + _make_lc_plan( + name="local/plan-two", + plan_id="01KHDE6WWS2171PWW3GJEBXZ8B", + phase=PlanPhase.EXECUTE, + state=ProcessingState.PROCESSING, + ), + ] + context.lc_mock.list_plans.return_value = context.lc_plans + + +@given("lifecycle coverage no plans exist for listing") +def step_lc_no_plans(context: Context) -> None: + context.lc_mock.list_plans.return_value = [] + + +@when("I run lifecycle coverage plan lifecycle-list") +def step_lc_plan_list(context: Context) -> None: + context.lc_result = context.lc_runner.invoke(plan_app, ["lifecycle-list"]) + + +@when('I run lifecycle coverage plan lifecycle-list with format "{fmt}"') +def step_lc_plan_list_fmt(context: Context, fmt: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["lifecycle-list", "--format", fmt] + ) + + +@when('I run lifecycle coverage plan lifecycle-list with invalid phase "{phase}"') +def step_lc_plan_list_invalid_phase(context: Context, phase: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["lifecycle-list", "--phase", phase] + ) + + +@when('I run lifecycle coverage plan lifecycle-list with invalid state "{state}"') +def step_lc_plan_list_invalid_state(context: Context, state: str) -> None: + context.lc_result = context.lc_runner.invoke( + plan_app, ["lifecycle-list", "--state", state] + ) + + +@then("lifecycle coverage plan list should show table") +def step_lc_plan_list_table(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "Plans" in context.lc_result.output + + +@then("lifecycle coverage plan list should succeed") +def step_lc_plan_list_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan list should show no plans message") +def step_lc_plan_list_empty(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "No plans found" in context.lc_result.output + + +@then("lifecycle coverage plan list should abort") +def step_lc_plan_list_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Plan status +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage a specific plan exists for status") +def step_lc_specific_plan_status(context: Context) -> None: + plan = _make_lc_plan( + project_links=[ProjectLink(project_name="proj-a", alias="api")], + automation_profile=AutomationProfileRef( + profile_name="trusted", + provenance=AutomationProfileProvenance.PLAN, + ), + ) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan in strategize phase exists") +def step_lc_plan_strategize_phase(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.STRATEGIZE, state=ProcessingState.PROCESSING) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan in execute phase exists") +def step_lc_plan_execute_phase(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.PROCESSING) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan in apply phase exists") +def step_lc_plan_apply_phase(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.APPLY, state=ProcessingState.PROCESSING) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan with applied outcome exists") +def step_lc_plan_applied(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.APPLY, state=ProcessingState.APPLIED) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan with constrained outcome exists") +def step_lc_plan_constrained(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.APPLY, + state=ProcessingState.CONSTRAINED, + error_message="Cannot proceed within constraints", + ) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan with errored outcome exists") +def step_lc_plan_errored(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.APPLY, + state=ProcessingState.ERRORED, + error_message="Apply failed", + ) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a plan with cancelled outcome exists") +def step_lc_plan_cancelled(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.STRATEGIZE, + state=ProcessingState.CANCELLED, + error_message="User cancelled", + ) + context.lc_mock.get_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage plan status service error") +def step_lc_plan_status_error(context: Context) -> None: + context.lc_mock.get_plan.side_effect = CleverAgentsError("Status fetch failed") + context.lc_plan = _make_lc_plan() + + +@when("I run lifecycle coverage plan status with plan ID") +def step_lc_plan_status_by_id(context: Context) -> None: + plan_id = getattr(context, "lc_plan", None) + pid = plan_id.identity.plan_id if plan_id else _PLAN_ULID + context.lc_result = context.lc_runner.invoke(plan_app, ["status", pid]) + + +@when("I run lifecycle coverage plan status without ID") +def step_lc_plan_status_no_id(context: Context) -> None: + context.lc_result = context.lc_runner.invoke(plan_app, ["status"]) + + +@when('I run lifecycle coverage plan status with plan ID and format "{fmt}"') +def step_lc_plan_status_fmt(context: Context, fmt: str) -> None: + plan_id = context.lc_plan.identity.plan_id + context.lc_result = context.lc_runner.invoke( + plan_app, ["status", plan_id, "--format", fmt] + ) + + +@then("lifecycle coverage plan status should show details") +def step_lc_plan_status_details(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "Plan" in context.lc_result.output + + +@then("lifecycle coverage plan status should show table") +def step_lc_plan_status_table(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan status should show no plans") +def step_lc_plan_status_no_plans(context: Context) -> None: + assert context.lc_result.exit_code == 0 + assert "No v3 lifecycle plans found" in context.lc_result.output + + +@then("lifecycle coverage plan status should succeed") +def step_lc_plan_status_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0 + + +@then("lifecycle coverage plan status should abort") +def step_lc_plan_status_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +@then('lifecycle coverage plan status should show phase "{phase}"') +def step_lc_plan_status_phase(context: Context, phase: str) -> None: + assert context.lc_result.exit_code == 0 + output_lower = context.lc_result.output.lower() + assert phase.lower() in output_lower + + +@then('lifecycle coverage plan status should show state "{state}"') +def step_lc_plan_status_state(context: Context, state: str) -> None: + assert context.lc_result.exit_code == 0 + output_lower = context.lc_result.output.lower() + assert state.lower() in output_lower + + +@then("lifecycle coverage plan should be terminal") +def step_lc_plan_terminal(context: Context) -> None: + assert context.lc_plan.is_terminal + + +# --------------------------------------------------------------------------- +# Plan execute +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage a plan ready for execute exists") +def step_lc_plan_ready_execute(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED) + context.lc_mock.execute_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a single plan ready for auto-execute exists") +def step_lc_single_execute(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.STRATEGIZE, + state=ProcessingState.COMPLETE, + ) + context.lc_mock.list_plans.return_value = [plan] + executed = _make_lc_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED) + context.lc_mock.execute_plan.return_value = executed + context.lc_plan = plan + + +@given("lifecycle coverage no plans ready for execute") +def step_lc_no_execute(context: Context) -> None: + context.lc_mock.list_plans.return_value = [] + + +@given("lifecycle coverage multiple plans ready for execute") +def step_lc_multi_execute(context: Context) -> None: + plans = [ + _make_lc_plan( + name="local/plan-x", + plan_id="01KHDE6WWS2171PWW3GJEBXZ8C", + phase=PlanPhase.STRATEGIZE, + state=ProcessingState.COMPLETE, + ), + _make_lc_plan( + name="local/plan-y", + plan_id="01KHDE6WWS2171PWW3GJEBXZ8D", + phase=PlanPhase.STRATEGIZE, + state=ProcessingState.COMPLETE, + ), + ] + context.lc_mock.list_plans.return_value = plans + + +@given("lifecycle coverage a plan with invalid transition for execute") +def step_lc_invalid_transition(context: Context) -> None: + context.lc_mock.execute_plan.side_effect = InvalidPhaseTransitionError( + PlanPhase.APPLY, PlanPhase.EXECUTE + ) + context.lc_plan = _make_lc_plan() + + +@given("lifecycle coverage a plan not ready for execute") +def step_lc_plan_not_ready(context: Context) -> None: + context.lc_mock.execute_plan.side_effect = PlanNotReadyError( + _PLAN_ULID, PlanPhase.STRATEGIZE, ProcessingState.QUEUED + ) + context.lc_plan = _make_lc_plan() + + +@given("lifecycle coverage a plan execute with general error") +def step_lc_plan_exec_general_error(context: Context) -> None: + context.lc_mock.execute_plan.side_effect = CleverAgentsError("Execute failed") + context.lc_plan = _make_lc_plan() + + +@when("I run lifecycle coverage plan execute with plan ID") +def step_lc_plan_execute(context: Context) -> None: + pid = getattr(context, "lc_plan", None) + plan_id = pid.identity.plan_id if pid else _PLAN_ULID + context.lc_result = context.lc_runner.invoke(plan_app, ["execute", plan_id]) + + +@when("I run lifecycle coverage plan execute without ID") +def step_lc_plan_execute_no_id(context: Context) -> None: + context.lc_result = context.lc_runner.invoke(plan_app, ["execute"]) + + +@when('I run lifecycle coverage plan execute with plan ID and format "{fmt}"') +def step_lc_plan_execute_fmt(context: Context, fmt: str) -> None: + pid = context.lc_plan.identity.plan_id + context.lc_result = context.lc_runner.invoke( + plan_app, ["execute", pid, "--format", fmt] + ) + + +@then("lifecycle coverage plan execute should succeed") +def step_lc_plan_execute_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0, f"CLI failed: {context.lc_result.output}" + + +@then("lifecycle coverage plan execute should abort") +def step_lc_plan_execute_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Plan lifecycle-apply +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage a plan ready for apply exists") +def step_lc_plan_ready_apply(context: Context) -> None: + plan = _make_lc_plan(phase=PlanPhase.APPLY, state=ProcessingState.QUEUED) + context.lc_mock.apply_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a single plan ready for auto-apply exists") +def step_lc_single_apply(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.EXECUTE, + state=ProcessingState.COMPLETE, + ) + context.lc_mock.list_plans.return_value = [plan] + applied = _make_lc_plan(phase=PlanPhase.APPLY, state=ProcessingState.QUEUED) + context.lc_mock.apply_plan.return_value = applied + context.lc_plan = plan + + +@given("lifecycle coverage no plans ready for apply") +def step_lc_no_apply(context: Context) -> None: + context.lc_mock.list_plans.return_value = [] + + +@given("lifecycle coverage multiple plans ready for apply") +def step_lc_multi_apply(context: Context) -> None: + plans = [ + _make_lc_plan( + name="local/plan-m", + plan_id="01KHDE6WWS2171PWW3GJEBXZ8E", + phase=PlanPhase.EXECUTE, + state=ProcessingState.COMPLETE, + ), + _make_lc_plan( + name="local/plan-n", + plan_id="01KHDE6WWS2171PWW3GJEBXZ8F", + phase=PlanPhase.EXECUTE, + state=ProcessingState.COMPLETE, + ), + ] + context.lc_mock.list_plans.return_value = plans + + +@given("lifecycle coverage a plan apply with general error") +def step_lc_plan_apply_general_error(context: Context) -> None: + context.lc_mock.apply_plan.side_effect = CleverAgentsError("Apply failed") + context.lc_plan = _make_lc_plan() + + +@when("I run lifecycle coverage plan lifecycle-apply with plan ID") +def step_lc_plan_apply(context: Context) -> None: + pid = getattr(context, "lc_plan", None) + plan_id = pid.identity.plan_id if pid else _PLAN_ULID + context.lc_result = context.lc_runner.invoke(plan_app, ["lifecycle-apply", plan_id]) + + +@when("I run lifecycle coverage plan lifecycle-apply without ID") +def step_lc_plan_apply_no_id(context: Context) -> None: + context.lc_result = context.lc_runner.invoke(plan_app, ["lifecycle-apply"]) + + +@when('I run lifecycle coverage plan lifecycle-apply with plan ID and format "{fmt}"') +def step_lc_plan_apply_fmt(context: Context, fmt: str) -> None: + pid = context.lc_plan.identity.plan_id + context.lc_result = context.lc_runner.invoke( + plan_app, ["lifecycle-apply", pid, "--format", fmt] + ) + + +@then("lifecycle coverage plan apply should succeed") +def step_lc_plan_apply_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0, f"CLI failed: {context.lc_result.output}" + + +@then("lifecycle coverage plan apply should abort") +def step_lc_plan_apply_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 + + +# --------------------------------------------------------------------------- +# Plan cancel +# --------------------------------------------------------------------------- + + +@given("lifecycle coverage a cancellable plan exists") +def step_lc_cancellable_plan(context: Context) -> None: + plan = _make_lc_plan( + phase=PlanPhase.STRATEGIZE, + state=ProcessingState.CANCELLED, + ) + context.lc_mock.cancel_plan.return_value = plan + context.lc_plan = plan + + +@given("lifecycle coverage a terminal plan exists for cancel") +def step_lc_terminal_cancel(context: Context) -> None: + context.lc_mock.cancel_plan.side_effect = PlanError( + "Plan is already in terminal state" + ) + context.lc_plan = _make_lc_plan() + + +@given("lifecycle coverage a plan cancel with general error") +def step_lc_cancel_general_error(context: Context) -> None: + context.lc_mock.cancel_plan.side_effect = CleverAgentsError("Cancel failed") + context.lc_plan = _make_lc_plan() + + +@when('I run lifecycle coverage plan cancel with reason "{reason}" and format "{fmt}"') +def step_lc_plan_cancel_fmt(context: Context, reason: str, fmt: str) -> None: + pid = context.lc_plan.identity.plan_id + context.lc_result = context.lc_runner.invoke( + plan_app, ["cancel", pid, "--reason", reason, "--format", fmt] + ) + + +@when('I run lifecycle coverage plan cancel providing reason "{reason}"') +def step_lc_plan_cancel_reason(context: Context, reason: str) -> None: + pid = getattr(context, "lc_plan", None) + plan_id = pid.identity.plan_id if pid else _PLAN_ULID + context.lc_result = context.lc_runner.invoke( + plan_app, ["cancel", plan_id, "--reason", reason] + ) + + +@when("I run lifecycle coverage plan cancel without reason") +def step_lc_plan_cancel_no_reason(context: Context) -> None: + pid = context.lc_plan.identity.plan_id + context.lc_result = context.lc_runner.invoke(plan_app, ["cancel", pid]) + + +@then("lifecycle coverage plan cancel should succeed") +def step_lc_plan_cancel_ok(context: Context) -> None: + assert context.lc_result.exit_code == 0, f"CLI failed: {context.lc_result.output}" + + +@then('lifecycle coverage cancel output should contain "{text}"') +def step_lc_cancel_output(context: Context, text: str) -> None: + assert text in context.lc_result.output + + +@then("lifecycle coverage plan cancel should abort") +def step_lc_plan_cancel_abort(context: Context) -> None: + assert context.lc_result.exit_code != 0 diff --git a/implementation_plan.md b/implementation_plan.md index b95dea333..97e55b7aa 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -1639,21 +1639,21 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled - [X] Git [Jeff]: `git branch -d feature/m1-cli-formats` - [X] Quality [Jeff]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%. -- [ ] **COMMIT (Owner: Brent | Group: A4b.tests | Branch: feature/m1-cli-tests | Planned: Day 9 | Expected: Day 11) - Commit message: "test(cli): expand lifecycle command coverage"** - - [ ] Git [Brent]: `git checkout master` - - [ ] Git [Brent]: `git pull origin master` - - [ ] Git [Brent]: `git checkout -b feature/m1-cli-tests` - - [ ] Git [Brent]: `git fetch origin && git merge origin/master` (run before final tests and before commit) - - [ ] Tests (Behave) [Brent]: Add CLI coverage for action create/list/show/archive and plan use/list/status/execute/apply/cancel (success + error paths). - - [ ] Tests (Behave) [Brent]: Add scenarios for plan use with multi-project args, automation profile overrides, invariants, and actor overrides. - - [ ] Tests (Behave) [Brent]: Add scenarios that assert Action phase visibility and Apply terminal outcomes (`applied`, `constrained`, `errored`, `cancelled`) in `plan status` output. - - [ ] Tests (Behave) [Brent]: Add negative cases for missing config, invalid args, invalid project names, and unknown actions/resources. - - [ ] Tests (Robot) [Brent]: Add end-to-end Robot suite for action → plan → execute → apply on a local repo (sandbox + ChangeSet capture verified). - - [ ] Docs [Brent]: Update `docs/development/testing.md` with CLI suites + fixtures. - - [ ] Tests (ASV) [Brent]: Add `benchmarks/plan_cli_smoke_bench.py` for CLI argument parsing overhead. - - [ ] Quality [Brent]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes. - - [ ] Git [Brent]: `git add .` - - [ ] Git [Brent]: `git commit -m "test(cli): expand lifecycle command coverage"` +- [X] **COMMIT (Owner: Brent | Group: A4b.tests | Branch: feature/m1-cli-tests | Planned: Day 9 | Expected: Day 11) - Commit message: "test(cli): expand lifecycle command coverage"** + - [X] Git [Brent]: `git checkout master` + - [X] Git [Brent]: `git pull origin master` + - [X] Git [Brent]: `git checkout -b feature/m1-cli-tests` + - [X] Git [Brent]: `git fetch origin && git merge origin/master` (run before final tests and before commit) + - [X] Tests (Behave) [Brent]: Add CLI coverage for action create/list/show/archive and plan use/list/status/execute/apply/cancel (success + error paths). + - [X] Tests (Behave) [Brent]: Add scenarios for plan use with multi-project args, automation profile overrides, invariants, and actor overrides. + - [X] Tests (Behave) [Brent]: Add scenarios that assert Action phase visibility and Apply terminal outcomes (`applied`, `constrained`, `errored`, `cancelled`) in `plan status` output. + - [X] Tests (Behave) [Brent]: Add negative cases for missing config, invalid args, invalid project names, and unknown actions/resources. + - [X] Tests (Robot) [Brent]: Add end-to-end Robot suite for action → plan → execute → apply on a local repo (sandbox + ChangeSet capture verified). + - [X] Docs [Brent]: Update `docs/development/testing.md` with CLI suites + fixtures. + - [X] Tests (ASV) [Brent]: Add `benchmarks/plan_cli_smoke_bench.py` for CLI argument parsing overhead. + - [X] Quality [Brent]: Run `nox` (all default sessions, including benchmark), fix any errors if needed ensuring nox passes. + - [X] Git [Brent]: `git add .` + - [X] Git [Brent]: `git commit -m "test(cli): expand lifecycle command coverage"` - [ ] Forgejo PR [Brent]: Open PR from `feature/m1-cli-tests` to `master` with description "Expand Behave + Robot CLI coverage for action/plan lifecycle commands and error paths.". - [ ] Git [Brent]: `git checkout master` - [ ] Git [Brent]: `git branch -d feature/m1-cli-tests` diff --git a/robot/cli_lifecycle_e2e.robot b/robot/cli_lifecycle_e2e.robot new file mode 100644 index 000000000..c16f31596 --- /dev/null +++ b/robot/cli_lifecycle_e2e.robot @@ -0,0 +1,73 @@ +*** Settings *** +Documentation End-to-end smoke tests for action -> plan -> execute -> apply lifecycle via CLI helpers +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_cli_lifecycle_e2e.py + +*** Test Cases *** +Action Create From Config Via CLI + [Documentation] Create an action from a YAML config file and verify it succeeds + ${result}= Run Process ${PYTHON} ${HELPER} action-create cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-action-create-ok + +Plan Use Creates Plan In Strategize Phase + [Documentation] Use an action to create a plan in Strategize phase + ${result}= Run Process ${PYTHON} ${HELPER} plan-use cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-use-ok + +Plan Execute Transitions To Execute Phase + [Documentation] Execute a plan, transitioning from Strategize to Execute + ${result}= Run Process ${PYTHON} ${HELPER} plan-execute cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-execute-ok + +Plan Lifecycle Apply Transitions To Apply Phase + [Documentation] Apply a plan, transitioning from Execute to Apply + ${result}= Run Process ${PYTHON} ${HELPER} plan-apply cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-apply-ok + +Plan Status Shows Plan Details + [Documentation] Verify plan status renders plan details + ${result}= Run Process ${PYTHON} ${HELPER} plan-status cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-status-ok + +Plan Cancel Cancels Non-Terminal Plan + [Documentation] Cancel a non-terminal plan + ${result}= Run Process ${PYTHON} ${HELPER} plan-cancel cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-cancel-ok + +Full Lifecycle Action To Apply + [Documentation] End-to-end: create action, use it, execute, apply + ${result}= Run Process ${PYTHON} ${HELPER} full-lifecycle cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-full-e2e-ok + +Plan Lifecycle List Shows Plans + [Documentation] Verify lifecycle-list renders plan table + ${result}= Run Process ${PYTHON} ${HELPER} plan-list cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-lifecycle-plan-list-ok diff --git a/robot/helper_cli_lifecycle_e2e.py b/robot/helper_cli_lifecycle_e2e.py new file mode 100644 index 000000000..8a4f7681e --- /dev/null +++ b/robot/helper_cli_lifecycle_e2e.py @@ -0,0 +1,335 @@ +"""Helper script for cli_lifecycle_e2e.robot end-to-end smoke tests. + +Each subcommand is self-contained and prints a sentinel on success. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +# Ensure local source tree is importable +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.action import app as action_app # noqa: E402 +from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 +from cleveragents.domain.models.core.action import Action, ActionState # noqa: E402 +from cleveragents.domain.models.core.plan import ( # noqa: E402 + AutomationLevel, + AutomationProfileProvenance, + AutomationProfileRef, + NamespacedName, + Plan, + PlanIdentity, + PlanPhase, + PlanTimestamps, + ProcessingState, + ProjectLink, +) + +runner = CliRunner() + +_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" + +_VALID_YAML = """\ +name: local/e2e-action +description: E2E lifecycle action +strategy_actor: openai/gpt-4 +execution_actor: openai/gpt-4 +definition_of_done: All e2e tests pass +""" + + +def _mock_action(name: str = "local/e2e-action") -> Action: + return Action( + namespaced_name=NamespacedName.parse(name), + description="E2E lifecycle action", + long_description=None, + definition_of_done="All e2e tests pass", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + state=ActionState.AVAILABLE, + reusable=True, + read_only=False, + created_at=datetime.now(), + updated_at=datetime.now(), + created_by=None, + ) + + +def _mock_plan( + name: str = "local/e2e-plan", + phase: PlanPhase = PlanPhase.STRATEGIZE, + state: ProcessingState = ProcessingState.QUEUED, + project_links: list[ProjectLink] | None = None, + plan_id: str = _PLAN_ULID, +) -> Plan: + now = datetime.now() + return Plan( + identity=PlanIdentity(plan_id=plan_id), + namespaced_name=NamespacedName.parse(name), + description="E2E lifecycle plan", + definition_of_done="Tests pass", + action_name="local/e2e-action", + phase=phase, + processing_state=state, + automation_level=AutomationLevel.MANUAL, + project_links=project_links or [ProjectLink(project_name="proj-a")], + arguments={"target_coverage": 80}, + arguments_order=["target_coverage"], + automation_profile=AutomationProfileRef( + profile_name="trusted", + provenance=AutomationProfileProvenance.PLAN, + ), + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + created_by=None, + timestamps=PlanTimestamps(created_at=now, updated_at=now), + ) + + +def _write_yaml(content: str) -> str: + fd, path = tempfile.mkstemp(suffix=".yaml") + with os.fdopen(fd, "w") as fh: + fh.write(content) + return path + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + + +def action_create() -> None: + """Verify action create from config file.""" + mock_service = MagicMock() + mock_service.create_action.return_value = _mock_action() + yaml_path = _write_yaml(_VALID_YAML) + + try: + with patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(action_app, ["create", "--config", yaml_path]) + if result.exit_code == 0: + print("cli-lifecycle-action-create-ok") + else: + print( + f"FAIL: action create returned {result.exit_code}", file=sys.stderr + ) + print(result.output, file=sys.stderr) + sys.exit(1) + finally: + os.unlink(yaml_path) + + +def plan_use() -> None: + """Verify plan use creates a plan in Strategize phase.""" + mock_service = MagicMock() + mock_service.get_action_by_name.return_value = _mock_action() + mock_service.use_action.return_value = _mock_plan() + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(plan_app, ["use", "local/e2e-action", "proj-a"]) + if result.exit_code == 0: + print("cli-lifecycle-plan-use-ok") + else: + print(f"FAIL: plan use returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def plan_execute() -> None: + """Verify plan execute transitions to Execute phase.""" + mock_service = MagicMock() + mock_service.execute_plan.return_value = _mock_plan( + phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED + ) + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(plan_app, ["execute", _PLAN_ULID]) + if result.exit_code == 0: + print("cli-lifecycle-plan-execute-ok") + else: + print(f"FAIL: plan execute returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def plan_apply() -> None: + """Verify plan lifecycle-apply transitions to Apply phase.""" + mock_service = MagicMock() + mock_service.apply_plan.return_value = _mock_plan( + phase=PlanPhase.APPLY, state=ProcessingState.QUEUED + ) + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID]) + if result.exit_code == 0: + print("cli-lifecycle-plan-apply-ok") + else: + print(f"FAIL: plan apply returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def plan_status() -> None: + """Verify plan status shows plan details.""" + mock_service = MagicMock() + mock_service.get_plan.return_value = _mock_plan( + project_links=[ProjectLink(project_name="proj-a", alias="api")] + ) + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(plan_app, ["status", _PLAN_ULID]) + if result.exit_code == 0: + print("cli-lifecycle-plan-status-ok") + else: + print(f"FAIL: plan status returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def plan_cancel() -> None: + """Verify plan cancel cancels a non-terminal plan.""" + mock_service = MagicMock() + mock_service.cancel_plan.return_value = _mock_plan( + phase=PlanPhase.STRATEGIZE, state=ProcessingState.CANCELLED + ) + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke( + plan_app, ["cancel", _PLAN_ULID, "--reason", "E2E test cancel"] + ) + if result.exit_code == 0: + print("cli-lifecycle-plan-cancel-ok") + else: + print(f"FAIL: plan cancel returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def plan_list() -> None: + """Verify lifecycle-list shows plan table.""" + mock_service = MagicMock() + mock_service.list_plans.return_value = [ + _mock_plan(name="local/plan-a", plan_id="01KHDE6WWS2171PWW3GJEBXZ8A"), + _mock_plan(name="local/plan-b", plan_id="01KHDE6WWS2171PWW3GJEBXZ8B"), + ] + + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ): + result = runner.invoke(plan_app, ["lifecycle-list"]) + if result.exit_code == 0: + print("cli-lifecycle-plan-list-ok") + else: + print(f"FAIL: plan list returned {result.exit_code}", file=sys.stderr) + print(result.output, file=sys.stderr) + sys.exit(1) + + +def full_lifecycle() -> None: + """End-to-end: action create -> plan use -> execute -> apply.""" + mock_service = MagicMock() + action = _mock_action() + mock_service.create_action.return_value = action + mock_service.get_action_by_name.return_value = action + + plan_strat = _mock_plan(phase=PlanPhase.STRATEGIZE, state=ProcessingState.QUEUED) + plan_exec = _mock_plan(phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED) + plan_apply = _mock_plan(phase=PlanPhase.APPLY, state=ProcessingState.QUEUED) + + mock_service.use_action.return_value = plan_strat + mock_service.execute_plan.return_value = plan_exec + mock_service.apply_plan.return_value = plan_apply + + yaml_path = _write_yaml(_VALID_YAML) + try: + with ( + patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=mock_service, + ), + patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_service, + ), + ): + # Step 1: Create action + r1 = runner.invoke(action_app, ["create", "--config", yaml_path]) + assert r1.exit_code == 0, f"action create failed: {r1.output}" + + # Step 2: Plan use + r2 = runner.invoke(plan_app, ["use", "local/e2e-action", "proj-a"]) + assert r2.exit_code == 0, f"plan use failed: {r2.output}" + + # Step 3: Plan execute + r3 = runner.invoke(plan_app, ["execute", _PLAN_ULID]) + assert r3.exit_code == 0, f"plan execute failed: {r3.output}" + + # Step 4: Plan apply + r4 = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID]) + assert r4.exit_code == 0, f"plan apply failed: {r4.output}" + + print("cli-lifecycle-full-e2e-ok") + + except AssertionError as exc: + print(f"FAIL: {exc}", file=sys.stderr) + sys.exit(1) + finally: + os.unlink(yaml_path) + + +# --------------------------------------------------------------------------- +# Main dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS = { + "action-create": action_create, + "plan-use": plan_use, + "plan-execute": plan_execute, + "plan-apply": plan_apply, + "plan-status": plan_status, + "plan-cancel": plan_cancel, + "plan-list": plan_list, + "full-lifecycle": full_lifecycle, +} + + +def main() -> None: + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>", file=sys.stderr) + sys.exit(2) + _COMMANDS[sys.argv[1]]() + + +if __name__ == "__main__": + main() -- 2.52.0 From 9db88a9541123d8c191e0ab5a620c31d999ad6e5 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Tue, 17 Feb 2026 23:38:45 +0000 Subject: [PATCH 2/2] fix(test): commit edge row so delete step sees it in its own session The "Deleting a resource with edges is rejected" scenario was failing because the ResourceEdgeModel row was flushed but never committed. Since ResourceRepository.delete() obtains its own session from the factory, the uncommitted edge was invisible and ResourceHasEdgesError was never raised. --- features/steps/resource_repository_steps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/features/steps/resource_repository_steps.py b/features/steps/resource_repository_steps.py index f3cc57452..b608db2bc 100644 --- a/features/steps/resource_repository_steps.py +++ b/features/steps/resource_repository_steps.py @@ -652,6 +652,7 @@ def step_parent_child_linked(context: Context, type_name: str) -> None: ) session.add(edge) session.flush() + session.commit() @when("the parent resource is deleted") -- 2.52.0