From 2c21611ad53a2576241ca091f617ae638b9b4e88 Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Tue, 24 Feb 2026 20:11:42 +0000 Subject: [PATCH] test(cli): cover action and plan extensions Added comprehensive test coverage for CLI extension features from #325: - Behave scenarios for automation_profile resolution, invariant ordering, and actor override error cases - Output snapshot assertions for JSON/YAML/table formats - Robot integration test for action show with optional actors/invariants - ASV benchmark for extended CLI scenario runtime baseline - Updated testing.md with CLI extension test fixture documentation ISSUES CLOSED: #326 --- benchmarks/cli_extension_tests_bench.py | 324 ++++++++++++++++++++++++ benchmarks/cli_extensions_bench.py | 7 + docs/development/testing.md | 130 ++++++++++ features/cli_extensions.feature | 172 +++++++++++++ features/steps/cli_extensions_steps.py | 183 +++++++++++++ robot/cli_extensions.robot | 32 +++ robot/helper_cli_extensions.py | 176 +++++++++++++ 7 files changed, 1024 insertions(+) create mode 100644 benchmarks/cli_extension_tests_bench.py diff --git a/benchmarks/cli_extension_tests_bench.py b/benchmarks/cli_extension_tests_bench.py new file mode 100644 index 000000000..f21ada12b --- /dev/null +++ b/benchmarks/cli_extension_tests_bench.py @@ -0,0 +1,324 @@ +"""ASV benchmarks for CLI extension test scenario runtime baselines. + +Benchmarks the extended test scenarios added in #326: +- Automation profile resolution across all builtin profiles +- Invariant ordering verification with multiple flags +- Actor override validation (valid and error paths) +- Output format rendering (JSON, YAML, table) with extended fields +""" + +from __future__ import annotations + +import importlib +import sys +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from typer.testing import CliRunner # noqa: E402 + +from cleveragents.cli.commands.action import app as action_app # noqa: E402 +from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 +from cleveragents.cli.commands.plan import ( # noqa: E402 + validate_namespaced_actor, +) +from cleveragents.domain.models.core.action import ( # noqa: E402 + Action, + ActionState, +) +from cleveragents.domain.models.core.plan import ( # noqa: E402 + AutomationProfileProvenance, + AutomationProfileRef, + InvariantSource, + NamespacedName, + Plan, + PlanIdentity, + PlanInvariant, + PlanPhase, + PlanTimestamps, + ProcessingState, + ProjectLink, +) + +_runner = CliRunner() +_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S" + +_BUILTIN_PROFILE_NAMES = [ + "manual", + "review", + "supervised", + "cautious", + "trusted", + "auto", +] + + +def _mock_action(name: str = "local/bench-action") -> Action: + return Action( + namespaced_name=NamespacedName.parse(name), + description="Benchmark action", + long_description=None, + definition_of_done="Benchmarks pass", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + estimation_actor="openai/gpt-4", + invariant_actor="anthropic/claude-3", + reusable=True, + read_only=False, + invariants=["No regressions", "Keep backward compat"], + inputs_schema={ + "type": "object", + "properties": {"coverage": {"type": "integer"}}, + }, + state=ActionState.AVAILABLE, + created_by=None, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + +def _mock_plan( + name: str = "local/bench-plan", + *, + with_profile: bool = False, + with_invariants: bool = False, + invariant_count: int = 2, +) -> Plan: + now = datetime.now() + profile = None + if with_profile: + profile = AutomationProfileRef( + profile_name="trusted", + provenance=AutomationProfileProvenance.PLAN, + ) + invariants: list[PlanInvariant] = [] + if with_invariants: + invariants = [ + PlanInvariant( + text=f"Invariant {i}", + source=InvariantSource.PLAN, + ) + for i in range(invariant_count) + ] + return Plan( + identity=PlanIdentity(plan_id=_PLAN_ULID), + namespaced_name=NamespacedName.parse(name), + description="Benchmark plan", + definition_of_done="Benchmarks pass", + action_name="local/bench-action", + phase=PlanPhase.STRATEGIZE, + processing_state=ProcessingState.QUEUED, + project_links=[ProjectLink(project_name="my-project")], + arguments={}, + arguments_order=[], + automation_profile=profile, + invariants=invariants, + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + created_by=None, + timestamps=PlanTimestamps(created_at=now, updated_at=now), + ) + + +class ProfileResolutionSuite: + """Benchmark automation profile resolution across all builtin profiles.""" + + def setup(self) -> None: + self._mock_service = MagicMock() + self._mock_service.get_action_by_name.return_value = _mock_action() + self._mock_service.use_action.return_value = _mock_plan(with_profile=True) + self._patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_resolve_all_builtin_profiles(self) -> None: + """Benchmark resolving each builtin profile in sequence.""" + for profile_name in _BUILTIN_PROFILE_NAMES: + _runner.invoke( + plan_app, + [ + "use", + "local/bench-action", + "--automation-profile", + profile_name, + ], + ) + + def time_resolve_single_profile(self) -> None: + """Benchmark resolving a single profile.""" + _runner.invoke( + plan_app, + ["use", "local/bench-action", "--automation-profile", "trusted"], + ) + + def time_reject_invalid_profile(self) -> None: + """Benchmark rejecting an invalid profile name.""" + _runner.invoke( + plan_app, + ["use", "local/bench-action", "--automation-profile", "nonexistent"], + ) + + +class InvariantOrderingSuite: + """Benchmark invariant ordering verification with multiple flags.""" + + def setup(self) -> None: + self._mock_service = MagicMock() + self._mock_service.get_action_by_name.return_value = _mock_action() + self._mock_service.use_action.return_value = _mock_plan( + with_invariants=True, invariant_count=5 + ) + self._patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_three_invariants(self) -> None: + """Benchmark plan use with three invariant flags.""" + _runner.invoke( + plan_app, + [ + "use", + "local/bench-action", + "--invariant", + "Alpha", + "--invariant", + "Beta", + "--invariant", + "Gamma", + ], + ) + + def time_five_invariants(self) -> None: + """Benchmark plan use with five invariant flags.""" + _runner.invoke( + plan_app, + [ + "use", + "local/bench-action", + "--invariant", + "A", + "--invariant", + "B", + "--invariant", + "C", + "--invariant", + "D", + "--invariant", + "E", + ], + ) + + +class ActorValidationErrorSuite: + """Benchmark actor override validation for error paths.""" + + def setup(self) -> None: + self._mock_service = MagicMock() + self._mock_service.get_action_by_name.return_value = _mock_action() + self._mock_service.use_action.return_value = _mock_plan() + self._patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._patcher.start() + + def teardown(self) -> None: + self._patcher.stop() + + def time_valid_actor_validation(self) -> None: + """Benchmark validating a valid actor name.""" + validate_namespaced_actor("openai/gpt-4", "--strategy-actor") + + def time_invalid_actor_rejection(self) -> None: + """Benchmark rejecting an invalid actor name.""" + try: + validate_namespaced_actor("bad-format", "--strategy-actor") + except Exception: + pass + + def time_plan_use_invalid_actor_cli(self) -> None: + """Benchmark CLI rejection of invalid actor flag.""" + _runner.invoke( + plan_app, + ["use", "local/bench-action", "--strategy-actor", "bad-format"], + ) + + +class OutputFormatRenderingSuite: + """Benchmark output rendering in JSON/YAML/table with extended fields.""" + + def setup(self) -> None: + self._mock_service = MagicMock() + self._mock_service.get_action_by_name.return_value = _mock_action() + plan = _mock_plan(with_profile=True, with_invariants=True) + self._mock_service.list_plans.return_value = [plan] + self._mock_service.get_plan.return_value = plan + self._plan_patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=self._mock_service, + ) + self._action_patcher = patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=self._mock_service, + ) + self._plan_patcher.start() + self._action_patcher.start() + + def teardown(self) -> None: + self._plan_patcher.stop() + self._action_patcher.stop() + + def time_action_show_json(self) -> None: + """Benchmark action show in JSON format.""" + _runner.invoke( + action_app, + ["show", "local/bench-action", "--format", "json"], + ) + + def time_action_show_yaml(self) -> None: + """Benchmark action show in YAML format.""" + _runner.invoke( + action_app, + ["show", "local/bench-action", "--format", "yaml"], + ) + + def time_action_show_table(self) -> None: + """Benchmark action show in table format.""" + _runner.invoke( + action_app, + ["show", "local/bench-action", "--format", "table"], + ) + + def time_plan_status_json(self) -> None: + """Benchmark plan status in JSON format.""" + _runner.invoke(plan_app, ["status", "--format", "json"]) + + def time_plan_status_yaml(self) -> None: + """Benchmark plan status in YAML format.""" + _runner.invoke(plan_app, ["status", "--format", "yaml"]) + + def time_plan_lifecycle_list_json(self) -> None: + """Benchmark plan lifecycle-list in JSON format.""" + _runner.invoke(plan_app, ["lifecycle-list", "--format", "json"]) diff --git a/benchmarks/cli_extensions_bench.py b/benchmarks/cli_extensions_bench.py index 5ef83386d..974b27ab1 100644 --- a/benchmarks/cli_extensions_bench.py +++ b/benchmarks/cli_extensions_bench.py @@ -55,17 +55,21 @@ def _mock_action(name: str = "local/bench-action") -> Action: return Action( namespaced_name=NamespacedName.parse(name), description="Benchmark action", + long_description=None, definition_of_done="Benchmarks pass", strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", estimation_actor="openai/gpt-4", invariant_actor="anthropic/claude-3", + reusable=True, + read_only=False, invariants=["No regressions", "Keep backward compat"], inputs_schema={ "type": "object", "properties": {"coverage": {"type": "integer"}}, }, state=ActionState.AVAILABLE, + created_by=None, created_at=datetime.now(), updated_at=datetime.now(), ) @@ -105,6 +109,9 @@ def _mock_plan( invariants=invariants, strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + created_by=None, timestamps=PlanTimestamps(created_at=now, updated_at=now), ) diff --git a/docs/development/testing.md b/docs/development/testing.md index 7826402e1..556dffa7c 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -714,3 +714,133 @@ nox -s benchmark | `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing | For full scale testing documentation, see `docs/development/scale_testing.md`. + +## CLI Extension Test Fixtures + +The CLI extension tests cover automation profile, invariant, and actor override +functionality added in #325. Deeper coverage added in #326 validates resolution +logic, ordering semantics, error cases, and output format snapshots. + +### Test Fixture Patterns + +The CLI extension test suites use a **mocked lifecycle service** pattern. The +`_get_lifecycle_service` function in both `plan.py` and `action.py` is patched +so that tests exercise real CLI argument parsing, validation, and output +rendering without requiring a database or running server. + +#### Common Fixtures + +| Fixture | Module | Description | +|---------|--------|-------------| +| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides | +| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) | +| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead | +| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations | + +#### Automation Profile Resolution Fixtures + +Tests validate that each builtin profile name (`manual`, `review`, `supervised`, +`cautious`, `trusted`, `auto`) resolves correctly through the CLI and that +invalid profile names are rejected with a helpful error listing available +profiles. The service mock returns a `Plan` with an `AutomationProfileRef` to +verify end-to-end profile assignment. + +#### Invariant Ordering Fixtures + +Tests pass multiple `--invariant` flags and verify the service receives them in +exact insertion order. This tests the list-accumulation behaviour of Typer's +repeated option handling. + +#### Actor Override Error Fixtures + +Tests exercise the `validate_namespaced_actor` regex +(`^[a-z][a-z0-9-]*/[a-z][a-z0-9._-]*$`) with various invalid inputs: +- Empty string +- Missing slash (`no-slash`) +- Uppercase characters (`UPPER/case`) +- Leading slash (`/leading-slash`) +- Trailing slash (`trail/ing/`) +- Double slashes (`open//ai`) +- Special characters (`open@ai/gpt-4`) +- Numeric-leading namespace (`123/model`) +- Whitespace (`open ai/gpt`) + +#### Output Snapshot Assertions + +Tests render plan status, lifecycle-list, and action show output in JSON, YAML, +and table formats, then assert that: +- JSON output is valid (parses without error) +- Required keys are present (`namespaced_name`, `phase`, `automation_profile`, etc.) +- Invariant text values survive serialization round-trips +- YAML output contains expected field names +- Table output contains human-readable field values + +### Behave Suite: `features/cli_extensions.feature` + +Covers scenarios across these areas: + +| Area | Scenarios | Description | +|------|-----------|-------------| +| Automation profile flags | 2 | Valid and invalid profile names | +| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty | +| Invariant flags | 2 | Single and dual invariant flags | +| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation | +| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors | +| Actor overrides (invalid) | 4 | Malformed namespace/name formats | +| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace | +| Plan status display | 4 | Table, JSON, single plan with invariants/profile | +| Plan lifecycle-list | 2 | Table and JSON with invariant count | +| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile | +| Combined flags | 1 | Profile + invariant together | +| validate_namespaced_actor | 2 | Unit-level accept/reject assertions | +| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action | + +Step definitions: `features/steps/cli_extensions_steps.py` + +### Robot Suite: `robot/cli_extensions.robot` + +Integration tests exercising CLI extensions through Python helper scripts: + +| Test Case | Description | +|-----------|-------------| +| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly | +| Plan Use With Automation Profile | Verifies `--automation-profile` works | +| Plan Use Actor Validation Valid | Valid namespaced actor format accepted | +| Plan Use Actor Validation Invalid | Invalid actor format rejected | +| Plan Use Combined Profile And Invariants | Both profile and invariants together | +| Action Show With Optional Actors And Invariants | `action show` renders estimation_actor, invariant_actor, invariants, inputs_schema | +| Action Show JSON With Optional Fields | JSON output includes all optional fields | +| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order | +| Profile Resolution All Builtins | All builtin profile names resolve correctly | + +Helper script: `robot/helper_cli_extensions.py` + +### ASV Benchmarks + +Two benchmark files cover CLI extension performance: + +**`benchmarks/cli_extensions_bench.py`** (original): +- `PlanUseWithProfileSuite` — plan use with automation profile +- `PlanUseWithInvariantsSuite` — plan use with invariant flags +- `PlanUseActorValidationSuite` — plan use with actor overrides +- `PlanStatusWithExtendedFieldsSuite` — status/lifecycle-list rendering +- `ActionShowExtendedSuite` — action show in rich and JSON formats + +**`benchmarks/cli_extension_tests_bench.py`** (extended #326): +- `ProfileResolutionSuite` — resolve all builtin profiles, single profile, invalid profile +- `InvariantOrderingSuite` — three and five invariant flag parsing +- `ActorValidationErrorSuite` — valid/invalid actor regex, CLI rejection +- `OutputFormatRenderingSuite` — JSON/YAML/table rendering for action show and plan status + +### Running CLI Extension Tests + +```bash +# Behave only (CLI extensions feature) +nox -s unit_tests -- features/cli_extensions.feature + +# Robot only (CLI extensions suite) +nox -s integration_tests -- --suite robot/cli_extensions.robot + +# Benchmarks +nox -s benchmark +``` diff --git a/features/cli_extensions.feature b/features/cli_extensions.feature index de06d0e30..c585efe45 100644 --- a/features/cli_extensions.feature +++ b/features/cli_extensions.feature @@ -189,3 +189,175 @@ Feature: CLI extensions for plan and action commands And validate_namespaced_actor rejects "UPPER/case" And validate_namespaced_actor rejects "/leading-slash" And validate_namespaced_actor rejects "trail/ing/" + + # ========================================================================== + # DEEP COVERAGE: Automation profile resolution logic (#326) + # ========================================================================== + + Scenario: Plan use resolves valid builtin profile "manual" + Given a cli extensions action exists + When I run plan use with automation profile flag "manual" + Then the cli extensions plan use should succeed + And the cli extensions plan should have automation profile "manual" + + Scenario: Plan use resolves valid builtin profile "review" + Given a cli extensions action exists + When I run plan use with automation profile flag "review" + Then the cli extensions plan use should succeed + And the cli extensions plan should have automation profile "review" + + Scenario: Plan use resolves valid builtin profile "supervised" + Given a cli extensions action exists + When I run plan use with automation profile flag "supervised" + Then the cli extensions plan use should succeed + And the cli extensions plan should have automation profile "supervised" + + Scenario: Plan use resolves valid builtin profile "cautious" + Given a cli extensions action exists + When I run plan use with automation profile flag "cautious" + Then the cli extensions plan use should succeed + And the cli extensions plan should have automation profile "cautious" + + Scenario: Plan use resolves valid builtin profile "auto" + Given a cli extensions action exists + When I run plan use with automation profile flag "auto" + Then the cli extensions plan use should succeed + And the cli extensions plan should have automation profile "auto" + + Scenario: Plan use rejects profile name with special characters + Given a cli extensions action exists + When I run plan use with automation profile flag "tr@sted!" + Then the cli extensions plan use should fail + + Scenario: Plan use rejects profile name with spaces + Given a cli extensions action exists + When I run plan use with automation profile flag "my profile" + Then the cli extensions plan use should fail + + Scenario: Plan use profile resolution provides available profiles on error + Given a cli extensions action exists + When I run plan use with automation profile flag "nonexistent" + Then the cli extensions plan use should fail + And the cli extensions error output mentions available profiles + + # ========================================================================== + # DEEP COVERAGE: Invariant ordering preservation (#326) + # ========================================================================== + + Scenario: Plan use with three invariants preserves insertion order + Given a cli extensions action exists + When I run plan use with three invariants "Alpha first" and "Beta second" and "Gamma third" + Then the cli extensions plan use should succeed + And the cli extensions service received invariants in order "Alpha first" then "Beta second" then "Gamma third" + + Scenario: Plan use with five invariants preserves insertion order + Given a cli extensions action exists + When I run plan use with five invariants "A" and "B" and "C" and "D" and "E" + Then the cli extensions plan use should succeed + And the cli extensions service received five invariants in exact order "A" "B" "C" "D" "E" + + # ========================================================================== + # DEEP COVERAGE: Actor override error cases (#326) + # ========================================================================== + + Scenario: Plan use rejects actor with double slashes + Given a cli extensions action exists + When I run plan use with strategy actor flag "open//ai" + Then the cli extensions plan use should fail with actor validation error + + Scenario: Plan use rejects actor with special characters in namespace + Given a cli extensions action exists + When I run plan use with strategy actor flag "open@ai/gpt-4" + Then the cli extensions plan use should fail with actor validation error + + Scenario: Plan use rejects actor with numeric-start namespace + Given a cli extensions action exists + When I run plan use with strategy actor flag "123/model" + Then the cli extensions plan use should fail with actor validation error + + Scenario: Plan use rejects actor with only a slash + Given a cli extensions action exists + When I run plan use with strategy actor flag "/" + Then the cli extensions plan use should fail with actor validation error + + Scenario: Plan use rejects actor with whitespace + Given a cli extensions action exists + When I run plan use with execution actor flag "open ai/gpt" + Then the cli extensions plan use should fail with actor validation error + + Scenario: Plan use rejects actor with missing name after slash + Given a cli extensions action exists + When I run plan use with estimation actor flag "openai/" + Then the cli extensions plan use should fail with actor validation error + + Scenario: validate_namespaced_actor rejects double slash + Then validate_namespaced_actor rejects "ns//name" + + Scenario: validate_namespaced_actor rejects special characters + Then validate_namespaced_actor rejects "ns@special/name" + And validate_namespaced_actor rejects "ns/name#bad" + + Scenario: validate_namespaced_actor rejects numeric-leading namespace + Then validate_namespaced_actor rejects "1abc/model" + + # ========================================================================== + # DEEP COVERAGE: Output snapshot assertions JSON/YAML/table (#326) + # ========================================================================== + + Scenario: Plan status in JSON format contains required keys + Given cli extensions plans exist with automation profile "trusted" + When I run cli extensions plan status with format "json" + Then the cli extensions output is valid JSON + And the cli extensions json output contains key "automation_profile" + And the cli extensions json output contains key "namespaced_name" + And the cli extensions json output contains key "phase" + + Scenario: Plan status in YAML format contains required fields + Given cli extensions plans exist with automation profile "trusted" + When I run cli extensions plan status with format "yaml" + Then the cli extensions output contains yaml field "automation_profile" + And the cli extensions output contains yaml field "namespaced_name" + + Scenario: Action show in JSON format contains all required keys + Given a cli extensions action with estimation actor "openai/gpt-4" + When I run cli extensions action show with format "json" + Then the cli extensions output is valid JSON + And the cli extensions json output contains key "estimation_actor" + And the cli extensions json output contains key "strategy_actor" + And the cli extensions json output contains key "execution_actor" + And the cli extensions json output contains key "description" + + Scenario: Action show in YAML format contains required fields + Given a cli extensions action with estimation actor "openai/gpt-4" + When I run cli extensions action show with format "yaml" + Then the cli extensions output contains yaml field "estimation_actor" + And the cli extensions output contains yaml field "strategy_actor" + + Scenario: Action show in JSON includes invariants list when present + Given a cli extensions action with invariants + When I run cli extensions action show with format "json" + Then the cli extensions output is valid JSON + And the cli extensions json output contains key "invariants" + And the cli extensions json invariants list has length 2 + + Scenario: Plan lifecycle-list in YAML shows invariant data + Given cli extensions plans exist with invariants + When I run cli extensions plan lifecycle-list with format "yaml" + Then the cli extensions output contains yaml field "invariants" + + Scenario: Plan lifecycle-list in JSON preserves invariant text + Given cli extensions plans exist with invariants + When I run cli extensions plan lifecycle-list with format "json" + Then the cli extensions output is valid JSON + And the cli extensions json output string contains "No warnings" + And the cli extensions json output string contains "Keep compat" + + Scenario: Action show in table format contains actor information + Given a cli extensions action with estimation actor "openai/gpt-4" + When I run cli extensions action show with format "table" + Then the cli extensions action output should contain "openai/gpt-4" + + Scenario: Plan status in table format contains profile name + Given cli extensions plans exist with automation profile "cautious" + When I run cli extensions plan status + Then the cli extensions status output should contain "cautious" diff --git a/features/steps/cli_extensions_steps.py b/features/steps/cli_extensions_steps.py index 218a0d5bd..e0617c2b6 100644 --- a/features/steps/cli_extensions_steps.py +++ b/features/steps/cli_extensions_steps.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from datetime import datetime from unittest.mock import MagicMock, patch @@ -621,6 +622,188 @@ def step_validate_actor_rejects(context: Context, actor: str) -> None: pass # Expected +# --------------------------------------------------------------------------- +# DEEP COVERAGE: Automation profile resolution (#326) +# --------------------------------------------------------------------------- + + +@then("the cli extensions error output mentions available profiles") +def step_error_mentions_available(context: Context) -> None: + """Verify error output lists available automation profiles.""" + output = context.last_result.output + # The error message should mention at least some builtin profiles + assert "Available" in output or "manual" in output or "trusted" in output, ( + f"Expected available profiles in error output: {output}" + ) + + +# --------------------------------------------------------------------------- +# DEEP COVERAGE: Invariant ordering (#326) +# --------------------------------------------------------------------------- + + +@when('I run plan use with three invariants "{inv1}" and "{inv2}" and "{inv3}"') +def step_run_plan_use_three_invariants( + context: Context, inv1: str, inv2: str, inv3: str +) -> None: + """Run plan use with three --invariant flags.""" + result = context.runner.invoke( + plan_app, + [ + "use", + "local/test-action", + "--invariant", + inv1, + "--invariant", + inv2, + "--invariant", + inv3, + ], + ) + context.last_result = result + + +@then( + "the cli extensions service received invariants in order " + '"{inv1}" then "{inv2}" then "{inv3}"' +) +def step_service_received_ordered_invariants( + context: Context, inv1: str, inv2: str, inv3: str +) -> None: + """Verify invariants were passed in exact insertion order.""" + call_args = context.mock_service.use_action.call_args + assert call_args is not None, "use_action was not called" + invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants") + assert invariants is not None, "No invariants passed to use_action" + texts = [inv.text for inv in invariants] + assert len(texts) >= 3, f"Expected at least 3 invariants, got {len(texts)}" + assert texts[0] == inv1, f"Expected '{inv1}' at index 0, got '{texts[0]}'" + assert texts[1] == inv2, f"Expected '{inv2}' at index 1, got '{texts[1]}'" + assert texts[2] == inv3, f"Expected '{inv3}' at index 2, got '{texts[2]}'" + + +@when( + "I run plan use with five invariants " + '"{inv1}" and "{inv2}" and "{inv3}" and "{inv4}" and "{inv5}"' +) +def step_run_plan_use_five_invariants( + context: Context, + inv1: str, + inv2: str, + inv3: str, + inv4: str, + inv5: str, +) -> None: + """Run plan use with five --invariant flags.""" + result = context.runner.invoke( + plan_app, + [ + "use", + "local/test-action", + "--invariant", + inv1, + "--invariant", + inv2, + "--invariant", + inv3, + "--invariant", + inv4, + "--invariant", + inv5, + ], + ) + context.last_result = result + + +@then( + "the cli extensions service received five invariants in exact order " + '"{inv1}" "{inv2}" "{inv3}" "{inv4}" "{inv5}"' +) +def step_service_received_five_ordered( + context: Context, + inv1: str, + inv2: str, + inv3: str, + inv4: str, + inv5: str, +) -> None: + """Verify five invariants in exact insertion order.""" + call_args = context.mock_service.use_action.call_args + assert call_args is not None, "use_action was not called" + invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants") + assert invariants is not None, "No invariants passed" + texts = [inv.text for inv in invariants] + expected = [inv1, inv2, inv3, inv4, inv5] + assert texts == expected, f"Expected {expected}, got {texts}" + + +# --------------------------------------------------------------------------- +# DEEP COVERAGE: Output snapshot assertions (#326) +# --------------------------------------------------------------------------- + + +@then("the cli extensions output is valid JSON") +def step_output_is_valid_json(context: Context) -> None: + """Verify last result output is valid JSON.""" + assert context.last_result.exit_code == 0, ( + f"Exit code: {context.last_result.exit_code}, " + f"output: {context.last_result.output}" + ) + output = context.last_result.output.strip() + try: + context.parsed_json = json.loads(output) + except json.JSONDecodeError as exc: + raise AssertionError( + f"Output is not valid JSON: {exc}\nOutput: {output}" + ) from exc + + +@then('the cli extensions json output contains key "{key}"') +def step_json_output_contains_key(context: Context, key: str) -> None: + """Verify parsed JSON contains a specific key.""" + parsed = getattr(context, "parsed_json", None) + if parsed is None: + # Parse if not already parsed + output = context.last_result.output.strip() + parsed = json.loads(output) + # Handle both dict and list-of-dict cases + if isinstance(parsed, list): + assert any(key in item for item in parsed if isinstance(item, dict)), ( + f"Key '{key}' not found in any list item" + ) + else: + assert key in parsed, f"Key '{key}' not in JSON: {list(parsed.keys())}" + + +@then('the cli extensions output contains yaml field "{field}"') +def step_output_contains_yaml_field(context: Context, field: str) -> None: + """Verify output contains a YAML field (key: value pattern).""" + assert context.last_result.exit_code == 0 + output = context.last_result.output + assert field in output, f"Expected YAML field '{field}' in output: {output}" + + +@then("the cli extensions json invariants list has length {length:d}") +def step_json_invariants_length(context: Context, length: int) -> None: + """Verify JSON invariants list has expected length.""" + parsed = getattr(context, "parsed_json", None) + if parsed is None: + output = context.last_result.output.strip() + parsed = json.loads(output) + invariants = parsed.get("invariants", []) + assert len(invariants) == length, ( + f"Expected {length} invariants, got {len(invariants)}: {invariants}" + ) + + +@then('the cli extensions json output string contains "{text}"') +def step_json_output_string_contains(context: Context, text: str) -> None: + """Verify the raw JSON output string contains the given text.""" + assert context.last_result.exit_code == 0 + output = context.last_result.output + assert text in output, f"Expected '{text}' in JSON output: {output}" + + # --------------------------------------------------------------------------- # Cleanup # --------------------------------------------------------------------------- diff --git a/robot/cli_extensions.robot b/robot/cli_extensions.robot index 1265dbc73..ea0685eff 100644 --- a/robot/cli_extensions.robot +++ b/robot/cli_extensions.robot @@ -47,3 +47,35 @@ Plan Use Combined Profile And Invariants Log ${result.stderr} Should Be Equal As Integers ${result.rc} 0 Should Contain ${result.stdout} cli-ext-plan-combined-ok + +Action Show With Optional Actors And Invariants + [Documentation] Verify action show includes optional actors, invariants, and inputs_schema when set + ${result}= Run Process ${PYTHON} ${HELPER} action-show-extended cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-ext-action-show-extended-ok + +Action Show JSON With Optional Fields + [Documentation] Verify action show JSON output includes optional fields + ${result}= Run Process ${PYTHON} ${HELPER} action-show-json cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-ext-action-show-json-ok + +Invariant Ordering Preserved + [Documentation] Verify that multiple invariant flags preserve insertion order + ${result}= Run Process ${PYTHON} ${HELPER} invariant-ordering cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-ext-invariant-ordering-ok + +Profile Resolution All Builtins + [Documentation] Verify that all builtin profiles resolve correctly + ${result}= Run Process ${PYTHON} ${HELPER} profile-resolution cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} cli-ext-profile-resolution-ok diff --git a/robot/helper_cli_extensions.py b/robot/helper_cli_extensions.py index 8d82ca434..5c352813e 100644 --- a/robot/helper_cli_extensions.py +++ b/robot/helper_cli_extensions.py @@ -17,6 +17,7 @@ if _SRC not in sys.path: from typer.testing import CliRunner # noqa: E402 +from cleveragents.cli.commands.action import app as action_app # noqa: E402 from cleveragents.cli.commands.plan import app as plan_app # noqa: E402 from cleveragents.domain.models.core.action import ( # noqa: E402 Action, @@ -62,6 +63,9 @@ def _mock_plan( invariants=invariants or [], strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", + reusable=True, + read_only=False, + created_by=None, timestamps=PlanTimestamps(created_at=now, updated_at=now), ) @@ -70,10 +74,14 @@ def _mock_action() -> Action: return Action( namespaced_name=NamespacedName.parse("local/test-action"), description="Test action for robot", + long_description=None, definition_of_done="All tests pass", strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", + reusable=True, + read_only=False, state=ActionState.AVAILABLE, + created_by=None, created_at=datetime.now(), updated_at=datetime.now(), ) @@ -228,6 +236,170 @@ def plan_use_combined() -> None: sys.exit(1) +def _mock_action_extended() -> Action: + """Create an action with all optional fields set for extended show tests.""" + return Action( + namespaced_name=NamespacedName.parse("local/test-action"), + description="Test action for robot extended", + long_description=None, + definition_of_done="All tests pass", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + estimation_actor="openai/gpt-4", + invariant_actor="anthropic/claude-3", + reusable=True, + read_only=False, + invariants=["No regressions", "Keep backward compat"], + inputs_schema={ + "type": "object", + "properties": {"coverage": {"type": "integer"}}, + }, + automation_profile="trusted", + state=ActionState.AVAILABLE, + created_by=None, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + +def action_show_extended() -> None: + """Verify action show renders optional actors, invariants, inputs_schema.""" + mock_svc = MagicMock() + mock_svc.get_action_by_name.return_value = _mock_action_extended() + with patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=mock_svc, + ): + result = runner.invoke(action_app, ["show", "local/test-action"]) + output = result.output + checks = [ + ("Estimation Actor" in output or "estimation_actor" in output), + ("Invariant Actor" in output or "invariant_actor" in output), + ("Invariants" in output or "invariants" in output), + ("Inputs Schema" in output or "inputs_schema" in output), + ] + if result.exit_code == 0 and all(checks): + print("cli-ext-action-show-extended-ok") + else: + print(f"FAIL: exit={result.exit_code} output={output}") + print(f"checks={checks}") + sys.exit(1) + + +def action_show_json() -> None: + """Verify action show JSON output includes optional fields.""" + import json + + mock_svc = MagicMock() + mock_svc.get_action_by_name.return_value = _mock_action_extended() + with patch( + "cleveragents.cli.commands.action._get_lifecycle_service", + return_value=mock_svc, + ): + result = runner.invoke( + action_app, ["show", "local/test-action", "--format", "json"] + ) + if result.exit_code != 0: + print(f"FAIL: exit={result.exit_code} output={result.output}") + sys.exit(1) + try: + data = json.loads(result.output.strip()) + except json.JSONDecodeError as exc: + print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}") + sys.exit(1) + required_keys = [ + "estimation_actor", + "invariant_actor", + "inputs_schema", + "invariants", + ] + missing = [k for k in required_keys if k not in data] + if missing: + print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}") + sys.exit(1) + print("cli-ext-action-show-json-ok") + + +def invariant_ordering() -> None: + """Verify that multiple invariant flags preserve insertion order.""" + mock_svc = MagicMock() + mock_svc.get_action_by_name.return_value = _mock_action() + mock_svc.use_action.return_value = _mock_plan( + invariants=[ + PlanInvariant(text="Alpha", source=InvariantSource.PLAN), + PlanInvariant(text="Beta", source=InvariantSource.PLAN), + PlanInvariant(text="Gamma", source=InvariantSource.PLAN), + ] + ) + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_svc, + ): + result = runner.invoke( + plan_app, + [ + "use", + "local/test-action", + "--invariant", + "Alpha", + "--invariant", + "Beta", + "--invariant", + "Gamma", + ], + ) + if result.exit_code != 0: + print(f"FAIL: exit={result.exit_code} output={result.output}") + sys.exit(1) + # Verify the service received invariants in order + call_args = mock_svc.use_action.call_args + invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants") + if invariants is None: + print("FAIL: no invariants passed to use_action") + sys.exit(1) + texts = [inv.text for inv in invariants] + if texts != ["Alpha", "Beta", "Gamma"]: + print(f"FAIL: ordering mismatch: {texts}") + sys.exit(1) + print("cli-ext-invariant-ordering-ok") + + +def profile_resolution() -> None: + """Verify all builtin profiles resolve correctly.""" + from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES + + for profile_name in sorted(BUILTIN_PROFILES): + mock_svc = MagicMock() + mock_svc.get_action_by_name.return_value = _mock_action() + mock_svc.use_action.return_value = _mock_plan( + automation_profile=AutomationProfileRef( + profile_name=profile_name, + provenance=AutomationProfileProvenance.PLAN, + ) + ) + with patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=mock_svc, + ): + result = runner.invoke( + plan_app, + [ + "use", + "local/test-action", + "--automation-profile", + profile_name, + ], + ) + if result.exit_code != 0: + print( + f"FAIL: profile '{profile_name}' " + f"exit={result.exit_code} " + f"output={result.output}" + ) + sys.exit(1) + print("cli-ext-profile-resolution-ok") + + # --------------------------------------------------------------------------- # Dispatcher # --------------------------------------------------------------------------- @@ -238,6 +410,10 @@ _COMMANDS: dict[str, object] = { "actor-valid": plan_use_actor_validation_valid, "actor-invalid": plan_use_actor_validation_invalid, "plan-combined": plan_use_combined, + "action-show-extended": action_show_extended, + "action-show-json": action_show_json, + "invariant-ordering": invariant_ordering, + "profile-resolution": profile_resolution, } if __name__ == "__main__":