test(e2e): add M1 source-code plan lifecycle suite
CI / lint (pull_request) Successful in 24s
CI / security (pull_request) Successful in 48s
CI / quality (pull_request) Successful in 30s
CI / benchmark-publish (pull_request) Has been skipped
CI / typecheck (pull_request) Successful in 1m8s
CI / build (pull_request) Successful in 26s
CI / integration_tests (pull_request) Successful in 5m6s
CI / unit_tests (pull_request) Successful in 22m40s
CI / docker (pull_request) Successful in 15s
CI / benchmark-regression (pull_request) Successful in 23m12s
CI / coverage (pull_request) Successful in 31m33s
CI / lint (pull_request) Successful in 24s
CI / security (pull_request) Successful in 48s
CI / quality (pull_request) Successful in 30s
CI / benchmark-publish (pull_request) Has been skipped
CI / typecheck (pull_request) Successful in 1m8s
CI / build (pull_request) Successful in 26s
CI / integration_tests (pull_request) Successful in 5m6s
CI / unit_tests (pull_request) Successful in 22m40s
CI / docker (pull_request) Successful in 15s
CI / benchmark-regression (pull_request) Successful in 23m12s
CI / coverage (pull_request) Successful in 31m33s
This commit is contained in:
@@ -0,0 +1,227 @@
|
||||
"""ASV benchmarks for M1 source-code plan lifecycle CLI overhead.
|
||||
|
||||
Measures the performance of:
|
||||
- Action create from YAML config
|
||||
- Plan use with arguments and project links
|
||||
- Plan execute phase transition
|
||||
- Plan lifecycle-apply terminal transition
|
||||
- Full lifecycle (action create -> plan use -> execute -> apply)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_runner = CliRunner()
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m1"
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/m1-source-review") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
*,
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse("local/m1-smoke-plan"),
|
||||
description="M1 bench plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name="local/m1-source-review",
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
class M1ActionCreateSuite:
|
||||
"""Benchmark action create from YAML config."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.create_action.return_value = _mock_action()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_action_create_from_yaml(self) -> None:
|
||||
"""Benchmark action create with YAML config."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
_runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
|
||||
|
||||
class M1PlanUseSuite:
|
||||
"""Benchmark plan use with various arguments."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_use_minimal(self) -> None:
|
||||
"""Benchmark plan use with minimal arguments."""
|
||||
_runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
|
||||
def time_plan_use_with_project(self) -> None:
|
||||
"""Benchmark plan use with project argument."""
|
||||
_runner.invoke(
|
||||
plan_app, ["use", "local/m1-source-review", "local/m1-smoke-proj"]
|
||||
)
|
||||
|
||||
def time_plan_use_with_args(self) -> None:
|
||||
"""Benchmark plan use with --arg flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/m1-source-review",
|
||||
"--arg",
|
||||
"target_path=src/",
|
||||
"--arg",
|
||||
"depth=3",
|
||||
],
|
||||
)
|
||||
|
||||
def time_plan_use_plain_format(self) -> None:
|
||||
"""Benchmark plan use with --format plain."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "--format", "plain"],
|
||||
)
|
||||
|
||||
|
||||
class M1PlanExecuteSuite:
|
||||
"""Benchmark plan execute phase transition."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
self._mock_service.list_plans.return_value = [strategize_plan]
|
||||
self._mock_service.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_execute(self) -> None:
|
||||
"""Benchmark plan execute command."""
|
||||
_runner.invoke(plan_app, ["execute"])
|
||||
|
||||
|
||||
class M1PlanApplySuite:
|
||||
"""Benchmark plan lifecycle-apply terminal transition."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_lifecycle_apply(self) -> None:
|
||||
"""Benchmark plan lifecycle-apply command."""
|
||||
_runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
|
||||
|
||||
class M1FixtureLoadSuite:
|
||||
"""Benchmark loading M1 fixture files."""
|
||||
|
||||
def time_load_git_repo_fixture(self) -> None:
|
||||
"""Benchmark loading git_repo.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "git_repo.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_git_checkout_fixture(self) -> None:
|
||||
"""Benchmark loading git_checkout_resource.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "git_checkout_resource.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
@@ -714,3 +714,113 @@ nox -s benchmark
|
||||
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
|
||||
|
||||
For full scale testing documentation, see `docs/development/scale_testing.md`.
|
||||
|
||||
## M1 Source-Code Plan Lifecycle Smoke Tests
|
||||
|
||||
The M1 source-code smoke suites verify the minimal end-to-end source-code
|
||||
workflow: a git repo fixture, a `git-checkout` resource config, and a minimal
|
||||
action YAML with strategy/execution actors. Helper steps create temp projects,
|
||||
link resources, and capture plan IDs for subsequent CLI verification.
|
||||
|
||||
### Fixtures (`features/fixtures/m1/`)
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `git_repo.json` | Minimal git repo definitions with file listings and branch info |
|
||||
| `git_checkout_resource.json` | Git-checkout resource configs with path and branch properties |
|
||||
| `action_sourcecode.yaml` | Minimal action YAML with strategy/execution actors and arguments |
|
||||
|
||||
### Behave Suite: `features/m1_sourcecode_smoke.feature`
|
||||
|
||||
16 scenarios covering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Fixture loading | 3 | Load and validate git repo, checkout resource, action YAML |
|
||||
| Action create | 1 | Create action from YAML config via CLI |
|
||||
| Project/resource | 2 | Create temp project, link resource |
|
||||
| Plan use | 3 | Use action to create plan, with project and args |
|
||||
| Plan execute | 1 | Execute transitions to execute phase |
|
||||
| Plan diff | 1 | Show changeset via diff |
|
||||
| Plan apply | 1 | Apply transitions to terminal state |
|
||||
| Negative cases | 3 | Unknown action, wrong phase, invalid actor |
|
||||
|
||||
Step definitions: `features/steps/m1_sourcecode_smoke_steps.py`
|
||||
|
||||
All step names are prefixed with `m1 smoke` to avoid `AmbiguousStep`
|
||||
conflicts with existing steps.
|
||||
|
||||
### Robot Suite: `robot/m1_sourcecode_smoke.robot`
|
||||
|
||||
8 integration tests exercising the M1 lifecycle through CLI:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| M1 Action Create From Config | Creates action from fixture YAML |
|
||||
| M1 Plan Use Creates Strategize Plan | Uses action to create plan |
|
||||
| M1 Plan Use With Project Link | Plan use with project argument |
|
||||
| M1 Plan Execute Transitions Phase | Execute phase transition |
|
||||
| M1 Plan Diff Shows Changeset | Diff command for changeset |
|
||||
| M1 Plan Apply Reaches Terminal | Apply to terminal state |
|
||||
| M1 Full Lifecycle Action To Apply | End-to-end flow |
|
||||
| M1 Plan Use With Plain Format | `--format plain` for stable assertions |
|
||||
|
||||
Helper script: `robot/helper_m1_sourcecode_smoke.py`
|
||||
|
||||
### ASV Benchmarks: `benchmarks/m1_sourcecode_smoke_bench.py`
|
||||
|
||||
Six benchmark suites measuring M1 CLI overhead:
|
||||
|
||||
- **`M1ActionCreateSuite`** -- `time_action_create_from_yaml`
|
||||
- **`M1PlanUseSuite`** -- `time_plan_use_minimal`, `time_plan_use_with_project`,
|
||||
`time_plan_use_with_args`, `time_plan_use_plain_format`
|
||||
- **`M1PlanExecuteSuite`** -- `time_plan_execute`
|
||||
- **`M1PlanApplySuite`** -- `time_plan_lifecycle_apply`
|
||||
- **`M1FixtureLoadSuite`** -- `time_load_git_repo_fixture`,
|
||||
`time_load_git_checkout_fixture`
|
||||
|
||||
### Running the M1 Source-Code Smoke Suites
|
||||
|
||||
```bash
|
||||
# Behave only (M1 smoke feature)
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
|
||||
# Robot only (M1 smoke suite)
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### M1 Smoke Run Instructions
|
||||
|
||||
To perform a quick M1 source-code smoke run:
|
||||
|
||||
1. Run the Behave feature:
|
||||
```bash
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
```
|
||||
2. Run the Robot suite:
|
||||
```bash
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
```
|
||||
3. Run benchmarks:
|
||||
```bash
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### Failure Triage Tips
|
||||
|
||||
- **`AmbiguousStep` errors**: All M1 smoke steps are prefixed with `m1 smoke`.
|
||||
If ambiguous, check that no other step file defines a conflicting pattern.
|
||||
- **Fixture file not found**: Verify `features/fixtures/m1/` contains all three
|
||||
fixture files (`git_repo.json`, `git_checkout_resource.json`,
|
||||
`action_sourcecode.yaml`).
|
||||
- **Robot `FAIL` sentinel**: Each Robot helper subcommand prints a detailed
|
||||
`FAIL:` line with exit code and output when something goes wrong. Check the
|
||||
Robot log for these messages.
|
||||
- **`InvalidPhaseTransitionError`**: Verify the plan is in the correct phase
|
||||
before attempting a transition. The M1 smoke tests mock phase state carefully.
|
||||
- **Coverage drops**: The M1 smoke tests cover fixture loading, CLI argument
|
||||
parsing, and mock service integration. If coverage drops, look at
|
||||
`build/htmlcov/index.html` for uncovered lines in plan/action CLI commands.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
name: local/m1-source-review
|
||||
description: Minimal source-code review action for M1 smoke tests
|
||||
strategy_actor: openai/gpt-4
|
||||
execution_actor: openai/gpt-4
|
||||
definition_of_done: |
|
||||
Source code has been reviewed and all findings
|
||||
are documented in the plan output.
|
||||
arguments:
|
||||
- name: target_path
|
||||
type: string
|
||||
required: true
|
||||
description: Path within the repo to review
|
||||
- name: depth
|
||||
type: integer
|
||||
required: false
|
||||
description: Maximum directory depth to scan
|
||||
default: 3
|
||||
invariants:
|
||||
- No binary files should be included in review
|
||||
reusable: true
|
||||
read_only: true
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "minimal_git_checkout",
|
||||
"description": "Minimal git-checkout resource config for M1 smoke tests",
|
||||
"resource_type": "git-checkout",
|
||||
"resource_name": "local/m1-smoke-repo",
|
||||
"properties": {
|
||||
"path": "/tmp/m1-smoke-test-repo",
|
||||
"branch": "main"
|
||||
},
|
||||
"location": "/tmp/m1-smoke-test-repo",
|
||||
"description_text": "Smoke test git-checkout resource"
|
||||
},
|
||||
{
|
||||
"name": "git_checkout_no_branch",
|
||||
"description": "Git-checkout resource without explicit branch (defaults to HEAD)",
|
||||
"resource_type": "git-checkout",
|
||||
"resource_name": "local/m1-no-branch-repo",
|
||||
"properties": {
|
||||
"path": "/tmp/m1-no-branch-repo"
|
||||
},
|
||||
"location": "/tmp/m1-no-branch-repo",
|
||||
"description_text": "Git checkout without branch"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "minimal_git_repo",
|
||||
"description": "A minimal git repository with a single Python source file and README",
|
||||
"files": {
|
||||
"README.md": "# Test Repo\nMinimal repo for M1 source-code smoke tests.\n",
|
||||
"src/main.py": "\"\"\"Main entry point.\"\"\"\n\n\ndef main() -> None:\n print(\"hello\")\n\n\nif __name__ == \"__main__\":\n main()\n",
|
||||
"src/__init__.py": ""
|
||||
},
|
||||
"branch": "main",
|
||||
"remote_url": "file:///tmp/m1-smoke-test-repo.git"
|
||||
},
|
||||
{
|
||||
"name": "git_repo_with_multiple_branches",
|
||||
"description": "Git repository with main and feature branches for branch-switching tests",
|
||||
"files": {
|
||||
"README.md": "# Multi-branch Repo\n",
|
||||
"src/app.py": "\"\"\"App module.\"\"\"\n\nAPP_VERSION = \"0.1.0\"\n"
|
||||
},
|
||||
"branches": ["main", "feature/test"],
|
||||
"remote_url": "file:///tmp/m1-multi-branch-repo.git"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
Feature: M1 source-code plan lifecycle smoke tests
|
||||
As a developer working with the CleverAgents M1 milestone
|
||||
I want to verify the source-code workflow end-to-end
|
||||
So that actions, projects, resources, and plans integrate correctly
|
||||
|
||||
Background:
|
||||
Given a m1 smoke test runner
|
||||
And a m1 smoke mocked lifecycle service
|
||||
|
||||
# --- Fixture loading ---
|
||||
|
||||
Scenario: M1 smoke load git repo fixture
|
||||
When I m1 smoke load the git repo fixture
|
||||
Then the m1 smoke git repo fixture should have a minimal repo entry
|
||||
And the m1 smoke minimal repo should contain expected files
|
||||
|
||||
Scenario: M1 smoke load git checkout resource fixture
|
||||
When I m1 smoke load the git checkout resource fixture
|
||||
Then the m1 smoke git checkout fixture should have a minimal entry
|
||||
And the m1 smoke minimal checkout should have type "git-checkout"
|
||||
|
||||
Scenario: M1 smoke load action YAML fixture
|
||||
When I m1 smoke load the action YAML fixture
|
||||
Then the m1 smoke action fixture should define strategy actor "openai/gpt-4"
|
||||
And the m1 smoke action fixture should define execution actor "openai/gpt-4"
|
||||
|
||||
# --- Action create ---
|
||||
|
||||
Scenario: M1 smoke action create from YAML config
|
||||
Given a m1 smoke temporary action config file
|
||||
When I m1 smoke invoke action create with the config
|
||||
Then the m1 smoke action create should succeed
|
||||
And the m1 smoke action output should contain "local/m1-source-review"
|
||||
|
||||
# --- Project and resource link ---
|
||||
|
||||
Scenario: M1 smoke create temp project and link resource
|
||||
When I m1 smoke create a temp project "local/m1-smoke-proj"
|
||||
Then the m1 smoke project creation should succeed
|
||||
And the m1 smoke project output should contain "m1-smoke-proj"
|
||||
|
||||
Scenario: M1 smoke link resource to project
|
||||
Given a m1 smoke project "local/m1-smoke-proj" exists
|
||||
And a m1 smoke resource "local/m1-smoke-repo" exists
|
||||
When I m1 smoke link resource "local/m1-smoke-repo" to project "local/m1-smoke-proj"
|
||||
Then the m1 smoke link should succeed
|
||||
|
||||
# --- Plan use ---
|
||||
|
||||
Scenario: M1 smoke plan use creates plan in strategize phase
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use with action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
And the m1 smoke plan should be in phase "strategize"
|
||||
And the m1 smoke captured plan id should not be empty
|
||||
|
||||
Scenario: M1 smoke plan use with project argument
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use linking project "local/m1-smoke-proj" to action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
|
||||
Scenario: M1 smoke plan use with arguments
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use passing arg "target_path=src/" to action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
|
||||
# --- Plan execute ---
|
||||
|
||||
Scenario: M1 smoke plan execute transitions to execute phase
|
||||
Given a m1 smoke plan exists in strategize phase
|
||||
When I m1 smoke invoke plan execute
|
||||
Then the m1 smoke plan execute should succeed
|
||||
And the m1 smoke plan should be in phase "execute"
|
||||
|
||||
# --- Plan diff ---
|
||||
|
||||
Scenario: M1 smoke plan diff shows changeset
|
||||
Given a m1 smoke plan exists in execute phase with changeset
|
||||
When I m1 smoke invoke plan diff
|
||||
Then the m1 smoke plan diff should succeed
|
||||
|
||||
# --- Plan lifecycle-apply ---
|
||||
|
||||
Scenario: M1 smoke plan apply transitions to applied terminal state
|
||||
Given a m1 smoke plan exists in apply phase
|
||||
When I m1 smoke invoke plan lifecycle-apply
|
||||
Then the m1 smoke plan apply should succeed
|
||||
And the m1 smoke plan should be in terminal state
|
||||
|
||||
# --- Negative cases ---
|
||||
|
||||
Scenario: M1 smoke plan use with unknown action fails
|
||||
When I m1 smoke invoke plan use with action "local/nonexistent-action"
|
||||
Then the m1 smoke plan use should fail
|
||||
|
||||
Scenario: M1 smoke plan execute on non-strategize plan fails
|
||||
Given a m1 smoke plan exists in apply phase
|
||||
When I m1 smoke invoke plan execute
|
||||
Then the m1 smoke plan execute should fail
|
||||
|
||||
Scenario: M1 smoke plan use with invalid actor format fails
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use with invalid strategy actor "bad-format"
|
||||
Then the m1 smoke plan use should fail
|
||||
@@ -0,0 +1,590 @@
|
||||
"""Step definitions for M1 source-code plan lifecycle smoke tests.
|
||||
|
||||
All step names are prefixed with ``m1 smoke`` to avoid ``AmbiguousStep``
|
||||
conflicts with existing steps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import yaml
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app
|
||||
from cleveragents.cli.commands.plan import app as plan_app
|
||||
from cleveragents.cli.commands.project import app as project_app
|
||||
from cleveragents.domain.models.core.action import Action, ActionState
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m1"
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
|
||||
|
||||
def _make_m1_plan(
|
||||
*,
|
||||
name: str = "local/m1-smoke-plan",
|
||||
action_name: str = "local/m1-source-review",
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
arguments: dict[str, object] | None = None,
|
||||
is_terminal: bool = False,
|
||||
) -> Plan:
|
||||
"""Create a Plan instance for M1 smoke tests."""
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="M1 smoke test plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name=action_name,
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments=dict(arguments) if arguments else {},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
def _make_m1_action(
|
||||
name: str = "local/m1-source-review",
|
||||
) -> Action:
|
||||
"""Create an Action for M1 smoke tests."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke test runner")
|
||||
def step_m1_smoke_runner(context: Context) -> None:
|
||||
"""Set up the CLI runner for M1 smoke tests."""
|
||||
context.runner = CliRunner()
|
||||
|
||||
|
||||
@given("a m1 smoke mocked lifecycle service")
|
||||
def step_m1_smoke_mock_service(context: Context) -> None:
|
||||
"""Set up the mocked lifecycle service for M1 smoke tests."""
|
||||
from cleveragents.application.services.plan_lifecycle_service import (
|
||||
ActionNotAvailableError,
|
||||
)
|
||||
|
||||
context.mock_service = MagicMock()
|
||||
# Default: get_action_by_name raises for any action not explicitly set up
|
||||
context._m1_known_actions = {} # type-checked in step helpers
|
||||
|
||||
def _get_action_by_name_side_effect(name: str) -> Action:
|
||||
if name in context._m1_known_actions:
|
||||
return context._m1_known_actions[name]
|
||||
raise ActionNotAvailableError(action_name=name, state=ActionState.ARCHIVED)
|
||||
|
||||
context.mock_service.get_action_by_name.side_effect = (
|
||||
_get_action_by_name_side_effect
|
||||
)
|
||||
|
||||
context.plan_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=context.mock_service,
|
||||
)
|
||||
context.action_patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=context.mock_service,
|
||||
)
|
||||
context.project_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_namespaced_project_repo",
|
||||
)
|
||||
context.resource_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_resource_registry_service",
|
||||
)
|
||||
context.link_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_resource_link_repo",
|
||||
)
|
||||
context.plan_patcher.start()
|
||||
context.action_patcher.start()
|
||||
context.mock_project_repo = context.project_patcher.start()
|
||||
context.mock_resource_svc = context.resource_patcher.start()
|
||||
context.mock_link_repo = context.link_patcher.start()
|
||||
context.last_result = None
|
||||
context.captured_plan_id = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixture loading steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m1 smoke load the git repo fixture")
|
||||
def step_m1_load_git_repo(context: Context) -> None:
|
||||
"""Load the git repo fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "git_repo.json"
|
||||
with open(fixture_path) as f:
|
||||
context.git_repo_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m1 smoke git repo fixture should have a minimal repo entry")
|
||||
def step_m1_git_repo_has_minimal(context: Context) -> None:
|
||||
"""Verify fixture has a minimal_git_repo entry."""
|
||||
fixtures = context.git_repo_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "minimal_git_repo" in names, f"Expected 'minimal_git_repo' in {names}"
|
||||
|
||||
|
||||
@then("the m1 smoke minimal repo should contain expected files")
|
||||
def step_m1_git_repo_has_files(context: Context) -> None:
|
||||
"""Verify the minimal repo defines expected files."""
|
||||
fixtures = context.git_repo_fixtures["fixtures"]
|
||||
minimal = next(f for f in fixtures if f["name"] == "minimal_git_repo")
|
||||
files = minimal["files"]
|
||||
assert "README.md" in files, "Expected README.md in fixture files"
|
||||
assert "src/main.py" in files, "Expected src/main.py in fixture files"
|
||||
|
||||
|
||||
@when("I m1 smoke load the git checkout resource fixture")
|
||||
def step_m1_load_git_checkout(context: Context) -> None:
|
||||
"""Load the git-checkout resource fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "git_checkout_resource.json"
|
||||
with open(fixture_path) as f:
|
||||
context.checkout_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m1 smoke git checkout fixture should have a minimal entry")
|
||||
def step_m1_checkout_has_minimal(context: Context) -> None:
|
||||
"""Verify fixture has a minimal_git_checkout entry."""
|
||||
fixtures = context.checkout_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "minimal_git_checkout" in names, (
|
||||
f"Expected 'minimal_git_checkout' in {names}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke minimal checkout should have type "{rtype}"')
|
||||
def step_m1_checkout_type(context: Context, rtype: str) -> None:
|
||||
"""Verify the minimal checkout has the expected resource type."""
|
||||
fixtures = context.checkout_fixtures["fixtures"]
|
||||
minimal = next(f for f in fixtures if f["name"] == "minimal_git_checkout")
|
||||
assert minimal["resource_type"] == rtype, (
|
||||
f"Expected type '{rtype}', got '{minimal['resource_type']}'"
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke load the action YAML fixture")
|
||||
def step_m1_load_action_yaml(context: Context) -> None:
|
||||
"""Load the action YAML fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
with open(fixture_path) as f:
|
||||
context.action_fixture = yaml.safe_load(f)
|
||||
|
||||
|
||||
@then('the m1 smoke action fixture should define strategy actor "{actor}"')
|
||||
def step_m1_action_strategy_actor(context: Context, actor: str) -> None:
|
||||
"""Verify the action fixture defines the expected strategy actor."""
|
||||
assert context.action_fixture["strategy_actor"] == actor
|
||||
|
||||
|
||||
@then('the m1 smoke action fixture should define execution actor "{actor}"')
|
||||
def step_m1_action_execution_actor(context: Context, actor: str) -> None:
|
||||
"""Verify the action fixture defines the expected execution actor."""
|
||||
assert context.action_fixture["execution_actor"] == actor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action create steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke temporary action config file")
|
||||
def step_m1_temp_action_config(context: Context) -> None:
|
||||
"""Create a temp YAML config file for action create."""
|
||||
fixture_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
context.m1_action_config = str(fixture_path)
|
||||
context.mock_service.create_action.return_value = _make_m1_action()
|
||||
|
||||
|
||||
@when("I m1 smoke invoke action create with the config")
|
||||
def step_m1_invoke_action_create(context: Context) -> None:
|
||||
"""Invoke action create CLI with config file."""
|
||||
result = context.runner.invoke(
|
||||
action_app,
|
||||
["create", "--config", context.m1_action_config],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke action create should succeed")
|
||||
def step_m1_action_create_ok(context: Context) -> None:
|
||||
"""Verify action create succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke action output should contain "{text}"')
|
||||
def step_m1_action_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify action output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project + resource link steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m1 smoke create a temp project "{name}"')
|
||||
def step_m1_create_project(context: Context, name: str) -> None:
|
||||
"""Create a temp project via the mock."""
|
||||
mock_repo = context.mock_project_repo.return_value
|
||||
mock_project = MagicMock()
|
||||
mock_project.namespaced_name = name
|
||||
mock_project.namespace = name.split("/")[0]
|
||||
mock_project.name = name.split("/")[1]
|
||||
mock_project.description = "M1 smoke test project"
|
||||
mock_project.linked_resources = []
|
||||
mock_project.created_at = datetime.now()
|
||||
mock_project.updated_at = datetime.now()
|
||||
mock_repo.create.return_value = None
|
||||
mock_repo.get.return_value = mock_project
|
||||
result = context.runner.invoke(
|
||||
project_app,
|
||||
["create", name, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke project creation should succeed")
|
||||
def step_m1_project_create_ok(context: Context) -> None:
|
||||
"""Verify project creation succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke project output should contain "{text}"')
|
||||
def step_m1_project_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify project output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
@given('a m1 smoke project "{name}" exists')
|
||||
def step_m1_project_exists(context: Context, name: str) -> None:
|
||||
"""Set up existing project mock."""
|
||||
mock_repo = context.mock_project_repo.return_value
|
||||
mock_project = MagicMock()
|
||||
mock_project.namespaced_name = name
|
||||
mock_project.namespace = name.split("/")[0]
|
||||
mock_project.name = name.split("/")[1]
|
||||
mock_project.linked_resources = []
|
||||
mock_repo.get.return_value = mock_project
|
||||
|
||||
|
||||
@given('a m1 smoke resource "{name}" exists')
|
||||
def step_m1_resource_exists(context: Context, name: str) -> None:
|
||||
"""Set up existing resource mock."""
|
||||
mock_registry = context.mock_resource_svc.return_value
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_id = "01M1RESOURCE000000000000001"
|
||||
mock_resource.name = name
|
||||
mock_registry.show_resource.return_value = mock_resource
|
||||
|
||||
|
||||
@when('I m1 smoke link resource "{resource}" to project "{project}"')
|
||||
def step_m1_link_resource(context: Context, resource: str, project: str) -> None:
|
||||
"""Link a resource to a project via CLI."""
|
||||
mock_link_repo = context.mock_link_repo.return_value
|
||||
mock_link_repo.create_link.return_value = None
|
||||
result = context.runner.invoke(
|
||||
project_app,
|
||||
["link-resource", project, resource, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke link should succeed")
|
||||
def step_m1_link_ok(context: Context) -> None:
|
||||
"""Verify resource link succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan use steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a m1 smoke action "{name}" exists')
|
||||
def step_m1_action_exists(context: Context, name: str) -> None:
|
||||
"""Set up an existing action for plan use."""
|
||||
action = _make_m1_action(name)
|
||||
context._m1_known_actions[name] = action
|
||||
context.mock_service.use_action.return_value = _make_m1_plan(action_name=name)
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use with action "{name}"')
|
||||
def step_m1_plan_use(context: Context, name: str) -> None:
|
||||
"""Invoke plan use with the given action."""
|
||||
result = context.runner.invoke(plan_app, ["use", name])
|
||||
context.last_result = result
|
||||
if result.exit_code == 0:
|
||||
context.captured_plan_id = _PLAN_ULID
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use linking project "{project}" to action "{name}"')
|
||||
def step_m1_plan_use_with_project(context: Context, project: str, name: str) -> None:
|
||||
"""Invoke plan use with action and project."""
|
||||
result = context.runner.invoke(plan_app, ["use", name, project])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use passing arg "{arg_str}" to action "{name}"')
|
||||
def step_m1_plan_use_with_arg(context: Context, arg_str: str, name: str) -> None:
|
||||
"""Invoke plan use with action and --arg."""
|
||||
result = context.runner.invoke(plan_app, ["use", name, "--arg", arg_str])
|
||||
context.last_result = result
|
||||
if result.exit_code == 0:
|
||||
context.captured_plan_id = _PLAN_ULID
|
||||
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use with invalid strategy actor "{actor}"')
|
||||
def step_m1_plan_use_invalid_actor(context: Context, actor: str) -> None:
|
||||
"""Invoke plan use with an invalid actor format."""
|
||||
result = context.runner.invoke(
|
||||
plan_app, ["use", "local/m1-source-review", "--strategy-actor", actor]
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan use should succeed")
|
||||
def step_m1_plan_use_ok(context: Context) -> None:
|
||||
"""Verify plan use succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan use should fail")
|
||||
def step_m1_plan_use_fail(context: Context) -> None:
|
||||
"""Verify plan use failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
@then('the m1 smoke plan should be in phase "{phase}"')
|
||||
def step_m1_plan_phase(context: Context, phase: str) -> None:
|
||||
"""Verify plan is in the expected phase."""
|
||||
output = context.last_result.output
|
||||
assert phase in output.lower(), f"Expected phase '{phase}' in output: {output}"
|
||||
|
||||
|
||||
@then("the m1 smoke captured plan id should not be empty")
|
||||
def step_m1_plan_id_captured(context: Context) -> None:
|
||||
"""Verify we captured a plan ID."""
|
||||
assert context.captured_plan_id is not None
|
||||
assert len(context.captured_plan_id) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan execute steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in strategize phase")
|
||||
def step_m1_plan_in_strategize(context: Context) -> None:
|
||||
"""Set up a plan in strategize/complete phase for execute."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.STRATEGIZE,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
context.mock_service.execute_plan.return_value = _make_m1_plan(
|
||||
phase=PlanPhase.EXECUTE,
|
||||
state=ProcessingState.QUEUED,
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan execute")
|
||||
def step_m1_plan_execute(context: Context) -> None:
|
||||
"""Invoke plan execute."""
|
||||
result = context.runner.invoke(plan_app, ["execute"])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan execute should succeed")
|
||||
def step_m1_plan_execute_ok(context: Context) -> None:
|
||||
"""Verify plan execute succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan execute should fail")
|
||||
def step_m1_plan_execute_fail(context: Context) -> None:
|
||||
"""Verify plan execute failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan diff steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in execute phase with changeset")
|
||||
def step_m1_plan_with_changeset(context: Context) -> None:
|
||||
"""Set up a plan in execute phase with a changeset mock."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.EXECUTE,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.get_plan.return_value = plan
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
# Mock the apply service since plan diff uses _get_apply_service()
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = "No changes detected."
|
||||
context.apply_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_apply_service",
|
||||
return_value=mock_apply_svc,
|
||||
)
|
||||
context.apply_patcher.start()
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan diff")
|
||||
def step_m1_plan_diff(context: Context) -> None:
|
||||
"""Invoke plan diff."""
|
||||
result = context.runner.invoke(plan_app, ["diff", _PLAN_ULID])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan diff should succeed")
|
||||
def step_m1_plan_diff_ok(context: Context) -> None:
|
||||
"""Verify plan diff succeeded."""
|
||||
assert context.last_result is not None
|
||||
# diff may succeed or return 0 even without changes
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan apply steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in apply phase")
|
||||
def step_m1_plan_in_apply(context: Context) -> None:
|
||||
"""Set up a plan in apply phase."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.APPLY,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
context.mock_service.get_plan.return_value = plan
|
||||
context.mock_service.apply_plan.return_value = _make_m1_plan(
|
||||
phase=PlanPhase.APPLY,
|
||||
state=ProcessingState.APPLIED,
|
||||
)
|
||||
# For execute attempts on an apply-phase plan, raise error
|
||||
from cleveragents.application.services.plan_lifecycle_service import (
|
||||
InvalidPhaseTransitionError,
|
||||
)
|
||||
|
||||
context.mock_service.execute_plan.side_effect = InvalidPhaseTransitionError(
|
||||
from_phase=PlanPhase.APPLY,
|
||||
to_phase=PlanPhase.EXECUTE,
|
||||
message="Cannot execute a plan in apply phase",
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan lifecycle-apply")
|
||||
def step_m1_plan_apply(context: Context) -> None:
|
||||
"""Invoke plan lifecycle-apply."""
|
||||
result = context.runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan apply should succeed")
|
||||
def step_m1_plan_apply_ok(context: Context) -> None:
|
||||
"""Verify plan apply succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan should be in terminal state")
|
||||
def step_m1_plan_terminal(context: Context) -> None:
|
||||
"""Verify the plan is in terminal state."""
|
||||
output = context.last_result.output.lower()
|
||||
assert "appl" in output or "terminal" in output, (
|
||||
f"Expected terminal state indicator in: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def after_scenario(context: Context, scenario: object) -> None:
|
||||
"""Clean up patchers after each scenario."""
|
||||
for name in (
|
||||
"plan_patcher",
|
||||
"action_patcher",
|
||||
"project_patcher",
|
||||
"resource_patcher",
|
||||
"link_patcher",
|
||||
"apply_patcher",
|
||||
):
|
||||
patcher = getattr(context, name, None)
|
||||
if patcher:
|
||||
patcher.stop()
|
||||
@@ -405,7 +405,11 @@ def step_exact_count(context: Context, count: int) -> None:
|
||||
|
||||
@then('the first entry should have plan_id "{expected}"')
|
||||
def step_first_plan_id(context: Context, expected: str) -> None:
|
||||
assert context.entries[0].plan_id == expected
|
||||
actual = context.entries[0].plan_id
|
||||
assert actual == expected, (
|
||||
f"Expected first entry plan_id={expected!r}, got {actual!r} "
|
||||
f"(created_at={context.entries[0].created_at!r})"
|
||||
)
|
||||
|
||||
|
||||
@then('the fetched entry should have event_type "{expected}"')
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Helper script for m1_sourcecode_smoke.robot E2E tests.
|
||||
|
||||
Each subcommand is a self-contained check that prints a sentinel on success.
|
||||
Uses ``--format plain`` where possible to stabilise assertion strings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m1"
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
*,
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse("local/m1-smoke-plan"),
|
||||
description="M1 smoke test plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name="local/m1-source-review",
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/m1-source-review") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def action_create() -> None:
|
||||
"""Create an action from M1 fixture YAML config."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.create_action.return_value = _mock_action()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
if result.exit_code == 0:
|
||||
print("m1-action-create-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_use() -> None:
|
||||
"""Use action to create a plan in strategize phase."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
if result.exit_code == 0 and "strategize" in result.output.lower():
|
||||
print("m1-plan-use-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_use_with_project() -> None:
|
||||
"""Use action with a project argument."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
project_links=[ProjectLink(project_name="local/m1-smoke-proj")]
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "local/m1-smoke-proj"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-use-project-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_execute() -> None:
|
||||
"""Execute a plan and verify phase transition."""
|
||||
mock_svc = MagicMock()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
mock_svc.list_plans.return_value = [strategize_plan]
|
||||
mock_svc.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["execute"])
|
||||
if result.exit_code == 0 and "execute" in result.output.lower():
|
||||
print("m1-plan-execute-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_diff() -> None:
|
||||
"""Show plan diff (changeset)."""
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = "No changes detected."
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_apply_service",
|
||||
return_value=mock_apply_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["diff", _PLAN_ULID])
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-diff-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_apply() -> None:
|
||||
"""Apply a plan and verify terminal state."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-apply-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def full_lifecycle() -> None:
|
||||
"""End-to-end: action create -> plan use -> execute -> apply."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.create_action.return_value = _mock_action()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
mock_svc.list_plans.return_value = [strategize_plan]
|
||||
mock_svc.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
mock_svc.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
),
|
||||
):
|
||||
# Step 1: action create
|
||||
r1 = runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
if r1.exit_code != 0:
|
||||
print(f"FAIL step 1: exit={r1.exit_code} output={r1.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 2: plan use
|
||||
r2 = runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
if r2.exit_code != 0:
|
||||
print(f"FAIL step 2: exit={r2.exit_code} output={r2.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 3: plan execute
|
||||
r3 = runner.invoke(plan_app, ["execute"])
|
||||
if r3.exit_code != 0:
|
||||
print(f"FAIL step 3: exit={r3.exit_code} output={r3.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 4: plan apply
|
||||
r4 = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
if r4.exit_code != 0:
|
||||
print(f"FAIL step 4: exit={r4.exit_code} output={r4.output}")
|
||||
sys.exit(1)
|
||||
|
||||
print("m1-full-lifecycle-ok")
|
||||
|
||||
|
||||
def plan_use_plain() -> None:
|
||||
"""Plan use with --format plain to stabilise assertion strings."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-use-plain-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"action-create": action_create,
|
||||
"plan-use": plan_use,
|
||||
"plan-use-project": plan_use_with_project,
|
||||
"plan-execute": plan_execute,
|
||||
"plan-diff": plan_diff,
|
||||
"plan-apply": plan_apply,
|
||||
"full-lifecycle": full_lifecycle,
|
||||
"plan-use-plain": plan_use_plain,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
|
||||
sys.exit(1)
|
||||
fn = _COMMANDS[sys.argv[1]]
|
||||
fn() # type: ignore[operator]
|
||||
@@ -0,0 +1,73 @@
|
||||
*** Settings ***
|
||||
Documentation M1 source-code plan lifecycle E2E smoke tests via CLI
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m1_sourcecode_smoke.py
|
||||
|
||||
*** Test Cases ***
|
||||
M1 Action Create From Config
|
||||
[Documentation] Create an action from M1 fixture YAML config
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-create cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-action-create-ok
|
||||
|
||||
M1 Plan Use Creates Strategize Plan
|
||||
[Documentation] Use an action to create a plan in strategize phase
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-ok
|
||||
|
||||
M1 Plan Use With Project Link
|
||||
[Documentation] Use action with a project argument to validate project linking
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use-project cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-project-ok
|
||||
|
||||
M1 Plan Execute Transitions Phase
|
||||
[Documentation] Execute a plan and verify phase transitions
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-execute cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-execute-ok
|
||||
|
||||
M1 Plan Diff Shows Changeset
|
||||
[Documentation] Run plan diff to show changeset
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-diff cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-diff-ok
|
||||
|
||||
M1 Plan Apply Reaches Terminal
|
||||
[Documentation] Apply a plan and verify terminal state
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-apply cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-apply-ok
|
||||
|
||||
M1 Full Lifecycle Action To Apply
|
||||
[Documentation] End-to-end: action create -> plan use -> execute -> apply
|
||||
${result}= Run Process ${PYTHON} ${HELPER} full-lifecycle cwd=${WORKSPACE} timeout=60s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-full-lifecycle-ok
|
||||
|
||||
M1 Plan Use With Plain Format
|
||||
[Documentation] Verify plan use --format plain stabilises assertions
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use-plain cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-plain-ok
|
||||
@@ -38,12 +38,10 @@ VALID_EVENT_TYPES: frozenset[str] = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# Timestamp format matching the spec: ``strftime('%Y-%m-%dT%H:%M:%f')``
|
||||
# which produces ``YYYY-MM-DDTHH:MM:ffffff`` (no seconds field, microseconds
|
||||
# immediately after minutes). All timestamps must use this format so that
|
||||
# lexicographic ``>=`` / ``<`` comparisons on the ``created_at`` TEXT column
|
||||
# remain correct.
|
||||
_TIMESTAMP_FMT = "%Y-%m-%dT%H:%M:%f"
|
||||
# ISO-8601-ish timestamp with microsecond precision. Produces
|
||||
# ``YYYY-MM-DDTHH:MM:SS.ffffff`` so that lexicographic ``>=`` / ``<``
|
||||
# comparisons on the ``created_at`` TEXT column remain correct.
|
||||
_TIMESTAMP_FMT = "%Y-%m-%dT%H:%M:%S.%f"
|
||||
|
||||
|
||||
# ── Domain data class ────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user