@@ -8,6 +8,12 @@
|
||||
provider ordering. Budget warnings are emitted at 90% usage, and requests are blocked at 100%.
|
||||
Per-provider cost table includes default token cost estimates for offline reporting. Budget
|
||||
exhaustion events are persisted in plan metadata for auditability. (#324)
|
||||
- Added comprehensive Behave, Robot Framework, and ASV test coverage for CLI extension
|
||||
features including automation profile resolution, invariant ordering, actor override
|
||||
error cases, and output format snapshot assertions.
|
||||
- Added comprehensive E2E test suite for M2 (Actor Graphs + Tool Sources) epic covering
|
||||
actor YAML loading, skill registry, tool lifecycle, and MCP stub tool discovery with
|
||||
Behave BDD scenarios, Robot Framework integration tests, and ASV performance benchmarks.
|
||||
- Added plan-level and project-level advisory locking with configurable timeouts, re-entrant
|
||||
acquisition, conflict detection, lock renewal, graceful shutdown release, startup cleanup of
|
||||
expired locks, and diagnostics check for stale lock reporting. (#327)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Contributors
|
||||
|
||||
* Jeffrey Phillips Freeman <jeffrey.freeman@syncleus.com>
|
||||
* Brent E. Edwards <brent.edwards@cleverthis.com>
|
||||
* Luis Mendes <luis.p.mendes@gmail.com>
|
||||
|
||||
# Details
|
||||
@@ -8,4 +9,5 @@
|
||||
Below are some of the specific details of various contributions.
|
||||
|
||||
* Jeffrey Phillips Freeman has acted as Lead Developer, daily contributor, and Project Owner.
|
||||
* Brent E. Edwards has contributed quality assurance, test coverage, and CI pipeline improvements.
|
||||
* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc.
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
"""ASV benchmarks for CLI extension test scenario runtime baselines.
|
||||
|
||||
Benchmarks the extended test scenarios added in #326:
|
||||
- Automation profile resolution across all builtin profiles
|
||||
- Invariant ordering verification with multiple flags
|
||||
- Actor override validation (valid and error paths)
|
||||
- Output format rendering (JSON, YAML, table) with extended fields
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import ( # noqa: E402
|
||||
validate_namespaced_actor,
|
||||
)
|
||||
from cleveragents.core.exceptions import ValidationError # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
AutomationProfileProvenance,
|
||||
AutomationProfileRef,
|
||||
InvariantSource,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanInvariant,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_runner = CliRunner()
|
||||
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
||||
|
||||
_BUILTIN_PROFILE_NAMES = [
|
||||
"manual",
|
||||
"review",
|
||||
"supervised",
|
||||
"cautious",
|
||||
"trusted",
|
||||
"auto",
|
||||
]
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/bench-action") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark action",
|
||||
long_description=None,
|
||||
definition_of_done="Benchmarks pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
name: str = "local/bench-plan",
|
||||
*,
|
||||
with_profile: bool = False,
|
||||
with_invariants: bool = False,
|
||||
invariant_count: int = 2,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
profile = None
|
||||
if with_profile:
|
||||
profile = AutomationProfileRef(
|
||||
profile_name="trusted",
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
)
|
||||
invariants: list[PlanInvariant] = []
|
||||
if with_invariants:
|
||||
invariants = [
|
||||
PlanInvariant(
|
||||
text=f"Invariant {i}",
|
||||
source=InvariantSource.PLAN,
|
||||
)
|
||||
for i in range(invariant_count)
|
||||
]
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark plan",
|
||||
definition_of_done="Benchmarks pass",
|
||||
action_name="local/bench-action",
|
||||
phase=PlanPhase.STRATEGIZE,
|
||||
processing_state=ProcessingState.QUEUED,
|
||||
project_links=[ProjectLink(project_name="my-project")],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
automation_profile=profile,
|
||||
invariants=invariants,
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
class ProfileResolutionSuite:
|
||||
"""Benchmark automation profile resolution across all builtin profiles."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan(with_profile=True)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_resolve_all_builtin_profiles(self) -> None:
|
||||
"""Benchmark resolving each builtin profile in sequence."""
|
||||
for profile_name in _BUILTIN_PROFILE_NAMES:
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--automation-profile",
|
||||
profile_name,
|
||||
],
|
||||
)
|
||||
|
||||
def time_resolve_single_profile(self) -> None:
|
||||
"""Benchmark resolving a single profile."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--automation-profile", "trusted"],
|
||||
)
|
||||
|
||||
def time_reject_invalid_profile(self) -> None:
|
||||
"""Benchmark rejecting an invalid profile name."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--automation-profile", "nonexistent"],
|
||||
)
|
||||
|
||||
|
||||
class InvariantOrderingSuite:
|
||||
"""Benchmark invariant ordering verification with multiple flags."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan(
|
||||
with_invariants=True, invariant_count=5
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_three_invariants(self) -> None:
|
||||
"""Benchmark plan use with three invariant flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--invariant",
|
||||
"Alpha",
|
||||
"--invariant",
|
||||
"Beta",
|
||||
"--invariant",
|
||||
"Gamma",
|
||||
],
|
||||
)
|
||||
|
||||
def time_five_invariants(self) -> None:
|
||||
"""Benchmark plan use with five invariant flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/bench-action",
|
||||
"--invariant",
|
||||
"A",
|
||||
"--invariant",
|
||||
"B",
|
||||
"--invariant",
|
||||
"C",
|
||||
"--invariant",
|
||||
"D",
|
||||
"--invariant",
|
||||
"E",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class ActorValidationErrorSuite:
|
||||
"""Benchmark actor override validation for error paths."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_valid_actor_validation(self) -> None:
|
||||
"""Benchmark validating a valid actor name."""
|
||||
validate_namespaced_actor("openai/gpt-4", "--strategy-actor")
|
||||
|
||||
def time_invalid_actor_rejection(self) -> None:
|
||||
"""Benchmark rejecting an invalid actor name."""
|
||||
try:
|
||||
validate_namespaced_actor("bad-format", "--strategy-actor")
|
||||
except ValidationError:
|
||||
pass
|
||||
|
||||
def time_plan_use_invalid_actor_cli(self) -> None:
|
||||
"""Benchmark CLI rejection of invalid actor flag."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/bench-action", "--strategy-actor", "bad-format"],
|
||||
)
|
||||
|
||||
|
||||
class OutputFormatRenderingSuite:
|
||||
"""Benchmark output rendering in JSON/YAML/table with extended fields."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
plan = _mock_plan(with_profile=True, with_invariants=True)
|
||||
self._mock_service.list_plans.return_value = [plan]
|
||||
self._mock_service.get_plan.return_value = plan
|
||||
self._plan_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._action_patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._plan_patcher.start()
|
||||
self._action_patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._plan_patcher.stop()
|
||||
self._action_patcher.stop()
|
||||
|
||||
def time_action_show_json(self) -> None:
|
||||
"""Benchmark action show in JSON format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "json"],
|
||||
)
|
||||
|
||||
def time_action_show_yaml(self) -> None:
|
||||
"""Benchmark action show in YAML format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "yaml"],
|
||||
)
|
||||
|
||||
def time_action_show_table(self) -> None:
|
||||
"""Benchmark action show in table format."""
|
||||
_runner.invoke(
|
||||
action_app,
|
||||
["show", "local/bench-action", "--format", "table"],
|
||||
)
|
||||
|
||||
def time_plan_status_json(self) -> None:
|
||||
"""Benchmark plan status in JSON format."""
|
||||
_runner.invoke(plan_app, ["status", "--format", "json"])
|
||||
|
||||
def time_plan_status_yaml(self) -> None:
|
||||
"""Benchmark plan status in YAML format."""
|
||||
_runner.invoke(plan_app, ["status", "--format", "yaml"])
|
||||
|
||||
def time_plan_lifecycle_list_json(self) -> None:
|
||||
"""Benchmark plan lifecycle-list in JSON format."""
|
||||
_runner.invoke(plan_app, ["lifecycle-list", "--format", "json"])
|
||||
@@ -55,17 +55,21 @@ def _mock_action(name: str = "local/bench-action") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Benchmark action",
|
||||
long_description=None,
|
||||
definition_of_done="Benchmarks pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
@@ -105,6 +109,9 @@ def _mock_plan(
|
||||
invariants=invariants,
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
"""ASV benchmarks for M1 source-code plan lifecycle CLI overhead.
|
||||
|
||||
Measures the performance of:
|
||||
- Action create from YAML config
|
||||
- Plan use with arguments and project links
|
||||
- Plan execute phase transition
|
||||
- Plan lifecycle-apply terminal transition
|
||||
- Full lifecycle (action create -> plan use -> execute -> apply)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_runner = CliRunner()
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m1"
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/m1-source-review") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
*,
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse("local/m1-smoke-plan"),
|
||||
description="M1 bench plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name="local/m1-source-review",
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
class M1ActionCreateSuite:
|
||||
"""Benchmark action create from YAML config."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.create_action.return_value = _mock_action()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_action_create_from_yaml(self) -> None:
|
||||
"""Benchmark action create with YAML config."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
_runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
|
||||
|
||||
class M1PlanUseSuite:
|
||||
"""Benchmark plan use with various arguments."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.get_action_by_name.return_value = _mock_action()
|
||||
self._mock_service.use_action.return_value = _mock_plan()
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_use_minimal(self) -> None:
|
||||
"""Benchmark plan use with minimal arguments."""
|
||||
_runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
|
||||
def time_plan_use_with_project(self) -> None:
|
||||
"""Benchmark plan use with project argument."""
|
||||
_runner.invoke(
|
||||
plan_app, ["use", "local/m1-source-review", "local/m1-smoke-proj"]
|
||||
)
|
||||
|
||||
def time_plan_use_with_args(self) -> None:
|
||||
"""Benchmark plan use with --arg flags."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/m1-source-review",
|
||||
"--arg",
|
||||
"target_path=src/",
|
||||
"--arg",
|
||||
"depth=3",
|
||||
],
|
||||
)
|
||||
|
||||
def time_plan_use_plain_format(self) -> None:
|
||||
"""Benchmark plan use with --format plain."""
|
||||
_runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "--format", "plain"],
|
||||
)
|
||||
|
||||
|
||||
class M1PlanExecuteSuite:
|
||||
"""Benchmark plan execute phase transition."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
self._mock_service.list_plans.return_value = [strategize_plan]
|
||||
self._mock_service.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_execute(self) -> None:
|
||||
"""Benchmark plan execute command."""
|
||||
_runner.invoke(plan_app, ["execute"])
|
||||
|
||||
|
||||
class M1PlanApplySuite:
|
||||
"""Benchmark plan lifecycle-apply terminal transition."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self._mock_service = MagicMock()
|
||||
self._mock_service.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
self._patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=self._mock_service,
|
||||
)
|
||||
self._patcher.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self._patcher.stop()
|
||||
|
||||
def time_plan_lifecycle_apply(self) -> None:
|
||||
"""Benchmark plan lifecycle-apply command."""
|
||||
_runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
|
||||
|
||||
class M1FixtureLoadSuite:
|
||||
"""Benchmark loading M1 fixture files."""
|
||||
|
||||
def time_load_git_repo_fixture(self) -> None:
|
||||
"""Benchmark loading git_repo.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "git_repo.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
|
||||
def time_load_git_checkout_fixture(self) -> None:
|
||||
"""Benchmark loading git_checkout_resource.json fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "git_checkout_resource.json"
|
||||
with open(fixture_path) as f:
|
||||
json.load(f)
|
||||
@@ -0,0 +1,208 @@
|
||||
"""ASV benchmarks for M2 actor + tool source smoke suite.
|
||||
|
||||
Measures baseline runtime for:
|
||||
- Actor YAML loading from fixture
|
||||
- Skill registry resolution
|
||||
- Tool lifecycle overhead (discover/activate/execute/deactivate)
|
||||
- MCP stub server discovery and invocation
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Ensure the local *source* tree is importable even when ASV has an
|
||||
# older build of the package installed.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
# Make the MCP stub mock importable (lives in features/mocks/, not installed).
|
||||
_MOCKS = str(Path(__file__).resolve().parents[1] / "features" / "mocks")
|
||||
if _MOCKS not in sys.path:
|
||||
sys.path.insert(0, _MOCKS)
|
||||
|
||||
# Force-reload the top-level package so Python picks up the source tree
|
||||
# version instead of a potentially stale installed copy.
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.actor.loader import ActorLoader # noqa: E402
|
||||
from cleveragents.actor.schema import ActorConfigSchema # noqa: E402
|
||||
from cleveragents.domain.models.core.skill import Skill, SkillInlineTool # noqa: E402
|
||||
from cleveragents.domain.models.core.tool import ( # noqa: E402
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
)
|
||||
from cleveragents.skills.protocol import SkillDefinition, SkillMetadata # noqa: E402
|
||||
from cleveragents.skills.registry import SkillRegistry # noqa: E402
|
||||
from cleveragents.tool.context import ToolExecutionContext # noqa: E402
|
||||
from cleveragents.tool.lifecycle import ( # noqa: E402
|
||||
ToolDescriptor,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
)
|
||||
from mcp_stub_server import McpStubServer # noqa: E402
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m2"
|
||||
|
||||
|
||||
# ── Actor loading benchmarks ─────────────────────────────────────
|
||||
|
||||
|
||||
class TimeM2ActorLoading:
|
||||
"""Benchmark actor YAML loading from M2 fixtures."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self.fixture_path = _FIXTURES_DIR / "actors" / "m2_hierarchical_actor.yaml"
|
||||
|
||||
def time_load_hierarchical_actor(self) -> None:
|
||||
ActorConfigSchema.from_yaml_file(self.fixture_path)
|
||||
|
||||
def time_discover_from_fixture_dir(self) -> None:
|
||||
loader = ActorLoader(search_roots=[_FIXTURES_DIR / "actors"])
|
||||
loader.discover()
|
||||
|
||||
def time_actor_loader_cache_hit(self) -> None:
|
||||
loader = ActorLoader(search_roots=[_FIXTURES_DIR / "actors"])
|
||||
loader.discover()
|
||||
loader.discover() # Second call should be a cache hit
|
||||
|
||||
|
||||
# ── Skill registry benchmarks ────────────────────────────────────
|
||||
|
||||
|
||||
class TimeM2SkillRegistry:
|
||||
"""Benchmark skill registry operations."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self.skill = Skill(
|
||||
name="m2bench/file-ops",
|
||||
description="Benchmark skill pack",
|
||||
tool_refs=["builtin/read_file", "builtin/list_files"],
|
||||
anonymous_tools=[
|
||||
SkillInlineTool(
|
||||
description="Bench echo tool",
|
||||
source=ToolSource.CUSTOM,
|
||||
code="def echo(t: str) -> str: return t",
|
||||
timeout=300,
|
||||
),
|
||||
],
|
||||
)
|
||||
self.metadata = SkillMetadata.from_skill(self.skill)
|
||||
self.defn = SkillDefinition(skill=self.skill, metadata=self.metadata)
|
||||
|
||||
def time_register_skill(self) -> None:
|
||||
registry = SkillRegistry()
|
||||
registry.register(self.defn)
|
||||
|
||||
def time_resolve_tools(self) -> None:
|
||||
registry = SkillRegistry()
|
||||
registry.register(self.defn)
|
||||
registry.resolve_tools(self.skill.name)
|
||||
|
||||
def time_list_all_skills(self) -> None:
|
||||
registry = SkillRegistry()
|
||||
registry.register(self.defn)
|
||||
registry.list_all()
|
||||
|
||||
|
||||
# ── Tool lifecycle benchmarks ────────────────────────────────────
|
||||
|
||||
|
||||
class _BenchToolInstance:
|
||||
"""Minimal tool instance for benchmark."""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self._name = name
|
||||
|
||||
def discover(self) -> ToolDescriptor:
|
||||
return ToolDescriptor(
|
||||
name=self._name, description="bench tool", source="custom"
|
||||
)
|
||||
|
||||
def activate(self, ctx: ToolExecutionContext) -> None:
|
||||
pass
|
||||
|
||||
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
|
||||
return ToolResult(success=True, data={"echo": params.get("text", "")})
|
||||
|
||||
def deactivate(self, ctx: ToolExecutionContext) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TimeM2ToolLifecycle:
|
||||
"""Benchmark tool lifecycle overhead."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self.name = "m2bench/echo"
|
||||
self.tool = Tool(
|
||||
name=self.name,
|
||||
description="Benchmark tool",
|
||||
source=ToolSource.BUILTIN,
|
||||
tool_type=ToolType.TOOL,
|
||||
capability=ToolCapability(
|
||||
writes=False, read_only=True, checkpointable=False
|
||||
),
|
||||
timeout=300,
|
||||
)
|
||||
self.instance = _BenchToolInstance(self.name)
|
||||
|
||||
def time_register_and_discover(self) -> None:
|
||||
runtime = ToolRuntime()
|
||||
runtime.register_tool(self.tool, self.instance)
|
||||
runtime.discover(self.name)
|
||||
|
||||
def time_full_lifecycle(self) -> None:
|
||||
runtime = ToolRuntime()
|
||||
runtime.register_tool(self.tool, self.instance)
|
||||
ctx = ToolExecutionContext(plan_id="bench-plan")
|
||||
runtime.activate(self.name, ctx)
|
||||
runtime.execute(self.name, {"text": "bench"}, ctx)
|
||||
runtime.deactivate(self.name, ctx)
|
||||
|
||||
def time_execute_only(self) -> None:
|
||||
runtime = ToolRuntime()
|
||||
runtime.register_tool(self.tool, self.instance)
|
||||
ctx = ToolExecutionContext(plan_id="bench-plan")
|
||||
runtime.execute(self.name, {"text": "bench"}, ctx)
|
||||
|
||||
|
||||
# ── MCP stub benchmarks ─────────────────────────────────────────
|
||||
|
||||
|
||||
class TimeM2McpStub:
|
||||
"""Benchmark MCP stub server discovery and invocation."""
|
||||
|
||||
def setup(self) -> None:
|
||||
self.server = McpStubServer()
|
||||
self.server.start()
|
||||
|
||||
def teardown(self) -> None:
|
||||
self.server.stop()
|
||||
|
||||
def time_discover_tools(self) -> None:
|
||||
self.server.discover()
|
||||
|
||||
def time_invoke_search(self) -> None:
|
||||
self.server.invoke("mcp/search", {"query": "benchmark"})
|
||||
|
||||
def time_invoke_fetch(self) -> None:
|
||||
self.server.invoke("mcp/fetch", {"url": "http://example.com"})
|
||||
|
||||
def time_invoke_transform(self) -> None:
|
||||
self.server.invoke("mcp/transform", {"data": "hello", "format": "upper"})
|
||||
|
||||
|
||||
# Module-level instances for ASV discovery
|
||||
time_actor = TimeM2ActorLoading()
|
||||
time_skill = TimeM2SkillRegistry()
|
||||
time_tool = TimeM2ToolLifecycle()
|
||||
time_mcp = TimeM2McpStub()
|
||||
@@ -792,3 +792,301 @@ nox -s benchmark
|
||||
| `benchmarks/scale_fixture_bench.py` | ASV | 6 | Performance benchmarks for fixture processing |
|
||||
|
||||
For full scale testing documentation, see `docs/development/scale_testing.md`.
|
||||
|
||||
## CLI Extension Test Fixtures
|
||||
|
||||
The CLI extension tests cover automation profile, invariant, and actor override
|
||||
functionality added in #325. Deeper coverage added in #326 validates resolution
|
||||
logic, ordering semantics, error cases, and output format snapshots.
|
||||
|
||||
### Test Fixture Patterns
|
||||
|
||||
The CLI extension test suites use a **mocked lifecycle service** pattern. The
|
||||
`_get_lifecycle_service` function in both `plan.py` and `action.py` is patched
|
||||
so that tests exercise real CLI argument parsing, validation, and output
|
||||
rendering without requiring a database or running server.
|
||||
|
||||
#### Common Fixtures
|
||||
|
||||
| Fixture | Module | Description |
|
||||
|---------|--------|-------------|
|
||||
| `_make_plan()` | `cli_extensions_steps.py` | Creates a `Plan` instance with configurable automation profile, invariants, and actor overrides |
|
||||
| `_make_action()` | `cli_extensions_steps.py` | Creates an `Action` with configurable optional fields (estimation_actor, invariant_actor, inputs_schema, automation_profile) |
|
||||
| `CliRunner` | `typer.testing` | Invokes CLI commands in-process without subprocess overhead |
|
||||
| `MagicMock` service | `unittest.mock` | Mocks `PlanLifecycleService` for plan/action operations |
|
||||
|
||||
#### Automation Profile Resolution Fixtures
|
||||
|
||||
Tests validate that each builtin profile name (`manual`, `review`, `supervised`,
|
||||
`cautious`, `trusted`, `auto`) resolves correctly through the CLI and that
|
||||
invalid profile names are rejected with a helpful error listing available
|
||||
profiles. The service mock returns a `Plan` with an `AutomationProfileRef` to
|
||||
verify end-to-end profile assignment.
|
||||
|
||||
#### Invariant Ordering Fixtures
|
||||
|
||||
Tests pass multiple `--invariant` flags and verify the service receives them in
|
||||
exact insertion order. This tests the list-accumulation behaviour of Typer's
|
||||
repeated option handling.
|
||||
|
||||
#### Actor Override Error Fixtures
|
||||
|
||||
Tests exercise the `validate_namespaced_actor` regex
|
||||
(`^[a-z][a-z0-9-]*/[a-z][a-z0-9._-]*$`) with various invalid inputs:
|
||||
- Empty string
|
||||
- Missing slash (`no-slash`)
|
||||
- Uppercase characters (`UPPER/case`)
|
||||
- Leading slash (`/leading-slash`)
|
||||
- Trailing slash (`trail/ing/`)
|
||||
- Double slashes (`open//ai`)
|
||||
- Special characters (`open@ai/gpt-4`)
|
||||
- Numeric-leading namespace (`123/model`)
|
||||
- Whitespace (`open ai/gpt`)
|
||||
|
||||
#### Output Snapshot Assertions
|
||||
|
||||
Tests render plan status, lifecycle-list, and action show output in JSON, YAML,
|
||||
and table formats, then assert that:
|
||||
- JSON output is valid (parses without error)
|
||||
- Required keys are present (`namespaced_name`, `phase`, `automation_profile`, etc.)
|
||||
- Invariant text values survive serialization round-trips
|
||||
- YAML output contains expected field names
|
||||
- Table output contains human-readable field values
|
||||
|
||||
### Behave Suite: `features/cli_extensions.feature`
|
||||
|
||||
Covers scenarios across these areas:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Automation profile flags | 2 | Valid and invalid profile names |
|
||||
| Profile resolution (deep) | 9 | All builtin profiles + error cases with special chars, spaces, empty |
|
||||
| Invariant flags | 2 | Single and dual invariant flags |
|
||||
| Invariant ordering (deep) | 2 | Three and five invariant insertion-order preservation |
|
||||
| Actor overrides (valid) | 4 | Strategy, execution, estimation, invariant actors |
|
||||
| Actor overrides (invalid) | 4 | Malformed namespace/name formats |
|
||||
| Actor error cases (deep) | 11 | Empty, double-slash, special chars, numeric-leading, whitespace |
|
||||
| Plan status display | 4 | Table, JSON, single plan with invariants/profile |
|
||||
| Plan lifecycle-list | 2 | Table and JSON with invariant count |
|
||||
| Action show | 8 | Optional actors, invariants, inputs_schema, automation profile |
|
||||
| Combined flags | 1 | Profile + invariant together |
|
||||
| validate_namespaced_actor | 2 | Unit-level accept/reject assertions |
|
||||
| Output snapshots (deep) | 9 | JSON/YAML/table format validation for plan and action |
|
||||
|
||||
Step definitions: `features/steps/cli_extensions_steps.py`
|
||||
|
||||
### Robot Suite: `robot/cli_extensions.robot`
|
||||
|
||||
Integration tests exercising CLI extensions through Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Plan Use With Invariants | Verifies `--invariant` flags pass through correctly |
|
||||
| Plan Use With Automation Profile | Verifies `--automation-profile` works |
|
||||
| Plan Use Actor Validation Valid | Valid namespaced actor format accepted |
|
||||
| Plan Use Actor Validation Invalid | Invalid actor format rejected |
|
||||
| Plan Use Combined Profile And Invariants | Both profile and invariants together |
|
||||
| Action Show With Optional Actors And Invariants | `action show` renders estimation_actor, invariant_actor, invariants, inputs_schema |
|
||||
| Action Show JSON With Optional Fields | JSON output includes all optional fields |
|
||||
| Invariant Ordering Preserved | Multiple invariant flags preserve insertion order |
|
||||
| Profile Resolution All Builtins | All builtin profile names resolve correctly |
|
||||
|
||||
Helper script: `robot/helper_cli_extensions.py`
|
||||
|
||||
### ASV Benchmarks
|
||||
|
||||
Two benchmark files cover CLI extension performance:
|
||||
|
||||
**`benchmarks/cli_extensions_bench.py`** (original):
|
||||
- `PlanUseWithProfileSuite` — plan use with automation profile
|
||||
- `PlanUseWithInvariantsSuite` — plan use with invariant flags
|
||||
- `PlanUseActorValidationSuite` — plan use with actor overrides
|
||||
- `PlanStatusWithExtendedFieldsSuite` — status/lifecycle-list rendering
|
||||
- `ActionShowExtendedSuite` — action show in rich and JSON formats
|
||||
|
||||
**`benchmarks/cli_extension_tests_bench.py`** (extended #326):
|
||||
- `ProfileResolutionSuite` — resolve all builtin profiles, single profile, invalid profile
|
||||
- `InvariantOrderingSuite` — three and five invariant flag parsing
|
||||
- `ActorValidationErrorSuite` — valid/invalid actor regex, CLI rejection
|
||||
- `OutputFormatRenderingSuite` — JSON/YAML/table rendering for action show and plan status
|
||||
|
||||
### Running CLI Extension Tests
|
||||
|
||||
```bash
|
||||
# Behave only (CLI extensions feature)
|
||||
nox -s unit_tests -- features/cli_extensions.feature
|
||||
|
||||
# Robot only (CLI extensions suite)
|
||||
nox -s integration_tests -- --suite robot/cli_extensions.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
## M2 Actor + Tool Source Smoke Suite
|
||||
|
||||
The M2 smoke suite validates the foundation for the Actor Graphs + Tool Sources epic (#356). It covers actor YAML loading, skill registry, tool lifecycle, and MCP stub tool discovery.
|
||||
|
||||
### Fixtures
|
||||
|
||||
M2 fixtures live in `features/fixtures/m2/`:
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `m2_hierarchical_actor.yaml` | Graph-type actor with planner → executor → reviewer nodes |
|
||||
| `m2_skill_pack.yaml` | Skill pack with tool references and an inline tool |
|
||||
|
||||
### MCP Stub Server
|
||||
|
||||
The MCP stub server (`features/mocks/mcp_stub_server.py`) simulates MCP tool discovery and invocation in-process without requiring the real MCP adapter (#159). It exposes three deterministic tools: `mcp/search`, `mcp/fetch`, and `mcp/transform`.
|
||||
|
||||
Usage in tests:
|
||||
|
||||
```python
|
||||
from features.mocks.mcp_stub_server import McpStubServer
|
||||
|
||||
server = McpStubServer()
|
||||
server.start()
|
||||
tools = server.discover() # Returns 3 stub tools
|
||||
result = server.invoke("mcp/search", {"query": "hello"})
|
||||
server.stop()
|
||||
```
|
||||
|
||||
### Test Suites
|
||||
|
||||
| Suite | Framework | Scenarios | Description |
|
||||
|-------|-----------|-----------|-------------|
|
||||
| `features/m2_actor_tool_smoke.feature` | Behave | 10 | Actor loading, skill registry, tool lifecycle, MCP stub |
|
||||
| `robot/m2_actor_tool_smoke.robot` | Robot | 6 | CLI-level smoke for actor/skill/tool/MCP operations |
|
||||
| `benchmarks/m2_actor_tool_smoke_bench.py` | ASV | 12 | Baseline runtime for actor, skill, tool, and MCP operations |
|
||||
|
||||
### Running the M2 Smoke Suite
|
||||
|
||||
```bash
|
||||
# Behave only (M2 feature)
|
||||
nox -s unit_tests -- features/m2_actor_tool_smoke.feature
|
||||
|
||||
# Robot only (M2 suite)
|
||||
nox -s integration_tests -- --suite robot/m2_actor_tool_smoke.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### What Is NOT Tested (deferred to dependent issues)
|
||||
|
||||
- Real MCP adapter integration (#159, assigned to Aditya)
|
||||
- Agent skills loader (#160, assigned to Aditya)
|
||||
- Hierarchical actor compiler (#158, assigned to Jeff)
|
||||
|
||||
The MCP stub server provides a test seam that can be replaced with the real adapter once #159 lands.
|
||||
|
||||
## M1 Source-Code Plan Lifecycle Smoke Tests
|
||||
|
||||
The M1 source-code smoke suites verify the minimal end-to-end source-code
|
||||
workflow: a git repo fixture, a `git-checkout` resource config, and a minimal
|
||||
action YAML with strategy/execution actors. Helper steps create temp projects,
|
||||
link resources, and capture plan IDs for subsequent CLI verification.
|
||||
|
||||
### Fixtures (`features/fixtures/m1/`)
|
||||
|
||||
| Fixture File | Description |
|
||||
|---|---|
|
||||
| `git_repo.json` | Minimal git repo definitions with file listings and branch info |
|
||||
| `git_checkout_resource.json` | Git-checkout resource configs with path and branch properties |
|
||||
| `action_sourcecode.yaml` | Minimal action YAML with strategy/execution actors and arguments |
|
||||
|
||||
### Behave Suite: `features/m1_sourcecode_smoke.feature`
|
||||
|
||||
16 scenarios covering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Fixture loading | 3 | Load and validate git repo, checkout resource, action YAML |
|
||||
| Action create | 1 | Create action from YAML config via CLI |
|
||||
| Project/resource | 2 | Create temp project, link resource |
|
||||
| Plan use | 3 | Use action to create plan, with project and args |
|
||||
| Plan execute | 1 | Execute transitions to execute phase |
|
||||
| Plan diff | 1 | Show changeset via diff |
|
||||
| Plan apply | 1 | Apply transitions to terminal state |
|
||||
| Negative cases | 3 | Unknown action, wrong phase, invalid actor |
|
||||
|
||||
Step definitions: `features/steps/m1_sourcecode_smoke_steps.py`
|
||||
|
||||
All step names are prefixed with `m1 smoke` to avoid `AmbiguousStep`
|
||||
conflicts with existing steps.
|
||||
|
||||
### Robot Suite: `robot/m1_sourcecode_smoke.robot`
|
||||
|
||||
8 integration tests exercising the M1 lifecycle through CLI:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| M1 Action Create From Config | Creates action from fixture YAML |
|
||||
| M1 Plan Use Creates Strategize Plan | Uses action to create plan |
|
||||
| M1 Plan Use With Project Link | Plan use with project argument |
|
||||
| M1 Plan Execute Transitions Phase | Execute phase transition |
|
||||
| M1 Plan Diff Shows Changeset | Diff command for changeset |
|
||||
| M1 Plan Apply Reaches Terminal | Apply to terminal state |
|
||||
| M1 Full Lifecycle Action To Apply | End-to-end flow |
|
||||
| M1 Plan Use With Plain Format | `--format plain` for stable assertions |
|
||||
|
||||
Helper script: `robot/helper_m1_sourcecode_smoke.py`
|
||||
|
||||
### ASV Benchmarks: `benchmarks/m1_sourcecode_smoke_bench.py`
|
||||
|
||||
Six benchmark suites measuring M1 CLI overhead:
|
||||
|
||||
- **`M1ActionCreateSuite`** -- `time_action_create_from_yaml`
|
||||
- **`M1PlanUseSuite`** -- `time_plan_use_minimal`, `time_plan_use_with_project`,
|
||||
`time_plan_use_with_args`, `time_plan_use_plain_format`
|
||||
- **`M1PlanExecuteSuite`** -- `time_plan_execute`
|
||||
- **`M1PlanApplySuite`** -- `time_plan_lifecycle_apply`
|
||||
- **`M1FixtureLoadSuite`** -- `time_load_git_repo_fixture`,
|
||||
`time_load_git_checkout_fixture`
|
||||
|
||||
### Running the M1 Source-Code Smoke Suites
|
||||
|
||||
```bash
|
||||
# Behave only (M1 smoke feature)
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
|
||||
# Robot only (M1 smoke suite)
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### M1 Smoke Run Instructions
|
||||
|
||||
To perform a quick M1 source-code smoke run:
|
||||
|
||||
1. Run the Behave feature:
|
||||
```bash
|
||||
nox -s unit_tests -- features/m1_sourcecode_smoke.feature
|
||||
```
|
||||
2. Run the Robot suite:
|
||||
```bash
|
||||
nox -s integration_tests -- --suite robot/m1_sourcecode_smoke.robot
|
||||
```
|
||||
3. Run benchmarks:
|
||||
```bash
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
### Failure Triage Tips
|
||||
|
||||
- **`AmbiguousStep` errors**: All M1 smoke steps are prefixed with `m1 smoke`.
|
||||
If ambiguous, check that no other step file defines a conflicting pattern.
|
||||
- **Fixture file not found**: Verify `features/fixtures/m1/` contains all three
|
||||
fixture files (`git_repo.json`, `git_checkout_resource.json`,
|
||||
`action_sourcecode.yaml`).
|
||||
- **Robot `FAIL` sentinel**: Each Robot helper subcommand prints a detailed
|
||||
`FAIL:` line with exit code and output when something goes wrong. Check the
|
||||
Robot log for these messages.
|
||||
- **`InvalidPhaseTransitionError`**: Verify the plan is in the correct phase
|
||||
before attempting a transition. The M1 smoke tests mock phase state carefully.
|
||||
- **Coverage drops**: The M1 smoke tests cover fixture loading, CLI argument
|
||||
parsing, and mock service integration. If coverage drops, look at
|
||||
`build/htmlcov/index.html` for uncovered lines in plan/action CLI commands.
|
||||
|
||||
@@ -189,3 +189,175 @@ Feature: CLI extensions for plan and action commands
|
||||
And validate_namespaced_actor rejects "UPPER/case"
|
||||
And validate_namespaced_actor rejects "/leading-slash"
|
||||
And validate_namespaced_actor rejects "trail/ing/"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Automation profile resolution logic (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "manual"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "manual"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "manual"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "review"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "review"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "review"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "supervised"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "supervised"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "supervised"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "cautious"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "cautious"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "cautious"
|
||||
|
||||
Scenario: Plan use resolves valid builtin profile "auto"
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "auto"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions plan should have automation profile "auto"
|
||||
|
||||
Scenario: Plan use rejects profile name with special characters
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "tr@sted!"
|
||||
Then the cli extensions plan use should fail
|
||||
|
||||
Scenario: Plan use rejects profile name with spaces
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "my profile"
|
||||
Then the cli extensions plan use should fail
|
||||
|
||||
Scenario: Plan use profile resolution provides available profiles on error
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with automation profile flag "nonexistent"
|
||||
Then the cli extensions plan use should fail
|
||||
And the cli extensions error output mentions available profiles
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Invariant ordering preservation (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use with three invariants preserves insertion order
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with three invariants "Alpha first" and "Beta second" and "Gamma third"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions service received invariants in order "Alpha first" then "Beta second" then "Gamma third"
|
||||
|
||||
Scenario: Plan use with five invariants preserves insertion order
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with five invariants "A" and "B" and "C" and "D" and "E"
|
||||
Then the cli extensions plan use should succeed
|
||||
And the cli extensions service received five invariants in exact order "A" "B" "C" "D" "E"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Actor override error cases (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan use rejects actor with double slashes
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "open//ai"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with special characters in namespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "open@ai/gpt-4"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with numeric-start namespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "123/model"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with only a slash
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with strategy actor flag "/"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with whitespace
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with execution actor flag "open ai/gpt"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: Plan use rejects actor with missing name after slash
|
||||
Given a cli extensions action exists
|
||||
When I run plan use with estimation actor flag "openai/"
|
||||
Then the cli extensions plan use should fail with actor validation error
|
||||
|
||||
Scenario: validate_namespaced_actor rejects double slash
|
||||
Then validate_namespaced_actor rejects "ns//name"
|
||||
|
||||
Scenario: validate_namespaced_actor rejects special characters
|
||||
Then validate_namespaced_actor rejects "ns@special/name"
|
||||
And validate_namespaced_actor rejects "ns/name#bad"
|
||||
|
||||
Scenario: validate_namespaced_actor rejects numeric-leading namespace
|
||||
Then validate_namespaced_actor rejects "1abc/model"
|
||||
|
||||
# ==========================================================================
|
||||
# DEEP COVERAGE: Output snapshot assertions JSON/YAML/table (#326)
|
||||
# ==========================================================================
|
||||
|
||||
Scenario: Plan status in JSON format contains required keys
|
||||
Given cli extensions plans exist with automation profile "trusted"
|
||||
When I run cli extensions plan status with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "automation_profile"
|
||||
And the cli extensions json output contains key "namespaced_name"
|
||||
And the cli extensions json output contains key "phase"
|
||||
|
||||
Scenario: Plan status in YAML format contains required fields
|
||||
Given cli extensions plans exist with automation profile "trusted"
|
||||
When I run cli extensions plan status with format "yaml"
|
||||
Then the cli extensions output contains yaml field "automation_profile"
|
||||
And the cli extensions output contains yaml field "namespaced_name"
|
||||
|
||||
Scenario: Action show in JSON format contains all required keys
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "estimation_actor"
|
||||
And the cli extensions json output contains key "strategy_actor"
|
||||
And the cli extensions json output contains key "execution_actor"
|
||||
And the cli extensions json output contains key "description"
|
||||
|
||||
Scenario: Action show in YAML format contains required fields
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "yaml"
|
||||
Then the cli extensions output contains yaml field "estimation_actor"
|
||||
And the cli extensions output contains yaml field "strategy_actor"
|
||||
|
||||
Scenario: Action show in JSON includes invariants list when present
|
||||
Given a cli extensions action with invariants
|
||||
When I run cli extensions action show with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output contains key "invariants"
|
||||
And the cli extensions json invariants list has length 2
|
||||
|
||||
Scenario: Plan lifecycle-list in YAML shows invariant data
|
||||
Given cli extensions plans exist with invariants
|
||||
When I run cli extensions plan lifecycle-list with format "yaml"
|
||||
Then the cli extensions output contains yaml field "invariants"
|
||||
|
||||
Scenario: Plan lifecycle-list in JSON preserves invariant text
|
||||
Given cli extensions plans exist with invariants
|
||||
When I run cli extensions plan lifecycle-list with format "json"
|
||||
Then the cli extensions output is valid JSON
|
||||
And the cli extensions json output string contains "No warnings"
|
||||
And the cli extensions json output string contains "Keep compat"
|
||||
|
||||
Scenario: Action show in table format contains actor information
|
||||
Given a cli extensions action with estimation actor "openai/gpt-4"
|
||||
When I run cli extensions action show with format "table"
|
||||
Then the cli extensions action output should contain "openai/gpt-4"
|
||||
|
||||
Scenario: Plan status in table format contains profile name
|
||||
Given cli extensions plans exist with automation profile "cautious"
|
||||
When I run cli extensions plan status
|
||||
Then the cli extensions status output should contain "cautious"
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
name: local/m1-source-review
|
||||
description: Minimal source-code review action for M1 smoke tests
|
||||
strategy_actor: openai/gpt-4
|
||||
execution_actor: openai/gpt-4
|
||||
definition_of_done: |
|
||||
Source code has been reviewed and all findings
|
||||
are documented in the plan output.
|
||||
arguments:
|
||||
- name: target_path
|
||||
type: string
|
||||
required: true
|
||||
description: Path within the repo to review
|
||||
- name: depth
|
||||
type: integer
|
||||
required: false
|
||||
description: Maximum directory depth to scan
|
||||
default: 3
|
||||
invariants:
|
||||
- No binary files should be included in review
|
||||
reusable: true
|
||||
read_only: true
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "minimal_git_checkout",
|
||||
"description": "Minimal git-checkout resource config for M1 smoke tests",
|
||||
"resource_type": "git-checkout",
|
||||
"resource_name": "local/m1-smoke-repo",
|
||||
"properties": {
|
||||
"path": "/tmp/m1-smoke-test-repo",
|
||||
"branch": "main"
|
||||
},
|
||||
"location": "/tmp/m1-smoke-test-repo",
|
||||
"description_text": "Smoke test git-checkout resource"
|
||||
},
|
||||
{
|
||||
"name": "git_checkout_no_branch",
|
||||
"description": "Git-checkout resource without explicit branch (defaults to HEAD)",
|
||||
"resource_type": "git-checkout",
|
||||
"resource_name": "local/m1-no-branch-repo",
|
||||
"properties": {
|
||||
"path": "/tmp/m1-no-branch-repo"
|
||||
},
|
||||
"location": "/tmp/m1-no-branch-repo",
|
||||
"description_text": "Git checkout without branch"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"fixtures": [
|
||||
{
|
||||
"name": "minimal_git_repo",
|
||||
"description": "A minimal git repository with a single Python source file and README",
|
||||
"files": {
|
||||
"README.md": "# Test Repo\nMinimal repo for M1 source-code smoke tests.\n",
|
||||
"src/main.py": "\"\"\"Main entry point.\"\"\"\n\n\ndef main() -> None:\n print(\"hello\")\n\n\nif __name__ == \"__main__\":\n main()\n",
|
||||
"src/__init__.py": ""
|
||||
},
|
||||
"branch": "main",
|
||||
"remote_url": "file:///tmp/m1-smoke-test-repo.git"
|
||||
},
|
||||
{
|
||||
"name": "git_repo_with_multiple_branches",
|
||||
"description": "Git repository with main and feature branches for branch-switching tests",
|
||||
"files": {
|
||||
"README.md": "# Multi-branch Repo\n",
|
||||
"src/app.py": "\"\"\"App module.\"\"\"\n\nAPP_VERSION = \"0.1.0\"\n"
|
||||
},
|
||||
"branches": ["main", "feature/test"],
|
||||
"remote_url": "file:///tmp/m1-multi-branch-repo.git"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
name: m2test/hierarchical-workflow
|
||||
type: graph
|
||||
description: M2 smoke test hierarchical actor with planner and executor nodes
|
||||
version: "1.0"
|
||||
model: gpt-4
|
||||
context_view: full
|
||||
memory:
|
||||
enabled: true
|
||||
max_messages: 100
|
||||
context:
|
||||
include_files:
|
||||
- README.md
|
||||
exclude_patterns:
|
||||
- "**/__pycache__/**"
|
||||
route:
|
||||
nodes:
|
||||
- id: planner
|
||||
type: agent
|
||||
name: Planner
|
||||
description: Plans the next steps
|
||||
config:
|
||||
model: gpt-4
|
||||
prompt: You are a task planner
|
||||
- id: executor
|
||||
type: tool
|
||||
name: Executor
|
||||
description: Executes file operations
|
||||
config:
|
||||
tool_name: builtin/read_file
|
||||
- id: reviewer
|
||||
type: agent
|
||||
name: Reviewer
|
||||
description: Reviews output quality
|
||||
config:
|
||||
model: gpt-4
|
||||
prompt: You are a code reviewer
|
||||
edges:
|
||||
- from_node: planner
|
||||
to_node: executor
|
||||
- from_node: executor
|
||||
to_node: reviewer
|
||||
entry_node: planner
|
||||
exit_nodes:
|
||||
- reviewer
|
||||
@@ -0,0 +1,23 @@
|
||||
name: m2test/file-ops-pack
|
||||
description: M2 smoke test skill pack with file operation tools
|
||||
tools:
|
||||
- name: builtin/read_file
|
||||
description: Read contents of a file
|
||||
- name: builtin/list_files
|
||||
description: List files in a directory
|
||||
inline_tools:
|
||||
- name: m2_echo
|
||||
description: Echo input back for testing
|
||||
source: custom
|
||||
code: |
|
||||
def m2_echo(text: str) -> str:
|
||||
return f"echo: {text}"
|
||||
input_schema:
|
||||
type: object
|
||||
properties:
|
||||
text:
|
||||
type: string
|
||||
required:
|
||||
- text
|
||||
writes: false
|
||||
checkpointable: false
|
||||
@@ -0,0 +1,104 @@
|
||||
Feature: M1 source-code plan lifecycle smoke tests
|
||||
As a developer working with the CleverAgents M1 milestone
|
||||
I want to verify the source-code workflow end-to-end
|
||||
So that actions, projects, resources, and plans integrate correctly
|
||||
|
||||
Background:
|
||||
Given a m1 smoke test runner
|
||||
And a m1 smoke mocked lifecycle service
|
||||
|
||||
# --- Fixture loading ---
|
||||
|
||||
Scenario: M1 smoke load git repo fixture
|
||||
When I m1 smoke load the git repo fixture
|
||||
Then the m1 smoke git repo fixture should have a minimal repo entry
|
||||
And the m1 smoke minimal repo should contain expected files
|
||||
|
||||
Scenario: M1 smoke load git checkout resource fixture
|
||||
When I m1 smoke load the git checkout resource fixture
|
||||
Then the m1 smoke git checkout fixture should have a minimal entry
|
||||
And the m1 smoke minimal checkout should have type "git-checkout"
|
||||
|
||||
Scenario: M1 smoke load action YAML fixture
|
||||
When I m1 smoke load the action YAML fixture
|
||||
Then the m1 smoke action fixture should define strategy actor "openai/gpt-4"
|
||||
And the m1 smoke action fixture should define execution actor "openai/gpt-4"
|
||||
|
||||
# --- Action create ---
|
||||
|
||||
Scenario: M1 smoke action create from YAML config
|
||||
Given a m1 smoke temporary action config file
|
||||
When I m1 smoke invoke action create with the config
|
||||
Then the m1 smoke action create should succeed
|
||||
And the m1 smoke action output should contain "local/m1-source-review"
|
||||
|
||||
# --- Project and resource link ---
|
||||
|
||||
Scenario: M1 smoke create temp project and link resource
|
||||
When I m1 smoke create a temp project "local/m1-smoke-proj"
|
||||
Then the m1 smoke project creation should succeed
|
||||
And the m1 smoke project output should contain "m1-smoke-proj"
|
||||
|
||||
Scenario: M1 smoke link resource to project
|
||||
Given a m1 smoke project "local/m1-smoke-proj" exists
|
||||
And a m1 smoke resource "local/m1-smoke-repo" exists
|
||||
When I m1 smoke link resource "local/m1-smoke-repo" to project "local/m1-smoke-proj"
|
||||
Then the m1 smoke link should succeed
|
||||
|
||||
# --- Plan use ---
|
||||
|
||||
Scenario: M1 smoke plan use creates plan in strategize phase
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use with action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
And the m1 smoke plan should be in phase "strategize"
|
||||
And the m1 smoke captured plan id should not be empty
|
||||
|
||||
Scenario: M1 smoke plan use with project argument
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use linking project "local/m1-smoke-proj" to action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
|
||||
Scenario: M1 smoke plan use with arguments
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use passing arg "target_path=src/" to action "local/m1-source-review"
|
||||
Then the m1 smoke plan use should succeed
|
||||
|
||||
# --- Plan execute ---
|
||||
|
||||
Scenario: M1 smoke plan execute transitions to execute phase
|
||||
Given a m1 smoke plan exists in strategize phase
|
||||
When I m1 smoke invoke plan execute
|
||||
Then the m1 smoke plan execute should succeed
|
||||
And the m1 smoke plan should be in phase "execute"
|
||||
|
||||
# --- Plan diff ---
|
||||
|
||||
Scenario: M1 smoke plan diff shows changeset
|
||||
Given a m1 smoke plan exists in execute phase with changeset
|
||||
When I m1 smoke invoke plan diff
|
||||
Then the m1 smoke plan diff should succeed
|
||||
|
||||
# --- Plan lifecycle-apply ---
|
||||
|
||||
Scenario: M1 smoke plan apply transitions to applied terminal state
|
||||
Given a m1 smoke plan exists in apply phase
|
||||
When I m1 smoke invoke plan lifecycle-apply
|
||||
Then the m1 smoke plan apply should succeed
|
||||
And the m1 smoke plan should be in terminal state
|
||||
|
||||
# --- Negative cases ---
|
||||
|
||||
Scenario: M1 smoke plan use with unknown action fails
|
||||
When I m1 smoke invoke plan use with action "local/nonexistent-action"
|
||||
Then the m1 smoke plan use should fail
|
||||
|
||||
Scenario: M1 smoke plan execute on non-strategize plan fails
|
||||
Given a m1 smoke plan exists in apply phase
|
||||
When I m1 smoke invoke plan execute
|
||||
Then the m1 smoke plan execute should fail
|
||||
|
||||
Scenario: M1 smoke plan use with invalid actor format fails
|
||||
Given a m1 smoke action "local/m1-source-review" exists
|
||||
When I m1 smoke invoke plan use with invalid strategy actor "bad-format"
|
||||
Then the m1 smoke plan use should fail
|
||||
@@ -0,0 +1,86 @@
|
||||
Feature: M2 actor + tool source smoke suite
|
||||
As a CleverAgents developer
|
||||
I want to verify that actor loading, skill registry, tool lifecycle,
|
||||
and MCP stub tool discovery work end-to-end
|
||||
So that the M2 (Actor Graphs + Tool Sources) foundation is solid
|
||||
|
||||
# ── Actor YAML loading ───────────────────────────────────────────
|
||||
|
||||
Scenario: Load hierarchical actor from YAML fixture
|
||||
Given I load the M2 hierarchical actor fixture
|
||||
Then the M2 actor name should be "m2test/hierarchical-workflow"
|
||||
And the M2 actor type should be "graph"
|
||||
And the M2 actor should have 3 graph nodes
|
||||
And the M2 actor entry node should be "planner"
|
||||
And the M2 actor exit nodes should include "reviewer"
|
||||
|
||||
Scenario: Load hierarchical actor via ActorLoader from fixture directory
|
||||
Given I create an M2 actor loader from the fixture directory
|
||||
When I run M2 actor discovery
|
||||
Then the M2 loader should find 1 actors
|
||||
And the M2 loader should contain actor "m2test/hierarchical-workflow"
|
||||
|
||||
# ── Skill pack loading and registry ──────────────────────────────
|
||||
|
||||
Scenario: Load skill pack from YAML fixture
|
||||
Given I load the M2 skill pack fixture
|
||||
Then the M2 skill name should be "m2test/file-ops-pack"
|
||||
And the M2 skill should have 2 tool references
|
||||
And the M2 skill should have 1 inline tools
|
||||
|
||||
Scenario: Register skill pack and verify tool resolution
|
||||
Given I register the M2 skill pack in a skill registry
|
||||
When I resolve tools for M2 skill "m2test/file-ops-pack"
|
||||
Then the M2 resolved tools should include a tool_ref entry
|
||||
And the M2 resolved tools should include an inline entry
|
||||
|
||||
Scenario: Skill registry lists registered skills correctly
|
||||
Given I register the M2 skill pack in a skill registry
|
||||
When I list all M2 skills in the registry
|
||||
Then the M2 skill list should contain "m2test/file-ops-pack"
|
||||
And the M2 skill list should have 1 entries
|
||||
|
||||
Scenario: Invoke built-in tool through skill context
|
||||
Given I create an M2 skill context with sandbox
|
||||
When I register a tool invocation for "builtin/read_file" in the M2 context
|
||||
Then the M2 skill context should have 1 invocation records
|
||||
|
||||
# ── Tool lifecycle ───────────────────────────────────────────────
|
||||
|
||||
Scenario: Tool lifecycle discover activate execute deactivate
|
||||
Given I create an M2 tool runtime with a mock tool "m2test/echo"
|
||||
And I create an M2 execution context with plan "m2-plan-001"
|
||||
When I discover M2 tool "m2test/echo"
|
||||
Then the M2 discovered descriptor name should be "m2test/echo"
|
||||
When I activate M2 tool "m2test/echo"
|
||||
And I execute M2 tool "m2test/echo" with params text "hello"
|
||||
Then the M2 execution result should be successful
|
||||
When I deactivate M2 tool "m2test/echo"
|
||||
Then the M2 tool should be deactivated
|
||||
|
||||
Scenario: Tool registry registers and retrieves tools
|
||||
Given I create an M2 tool registry
|
||||
When I register an M2 tool spec "m2test/file-reader"
|
||||
Then getting M2 tool "m2test/file-reader" should return a spec
|
||||
And listing M2 tools should include "m2test/file-reader"
|
||||
|
||||
# ── MCP stub tool discovery and invocation ───────────────────────
|
||||
|
||||
Scenario: MCP stub tool discovery and invocation
|
||||
Given I start the M2 MCP stub server
|
||||
When I discover tools from the M2 MCP stub
|
||||
Then the M2 stub should expose 3 tools
|
||||
And the M2 stub tools should include "mcp/search"
|
||||
And the M2 stub tools should include "mcp/fetch"
|
||||
And the M2 stub tools should include "mcp/transform"
|
||||
When I invoke M2 MCP stub tool "mcp/search" with query "test"
|
||||
Then the M2 MCP stub result should contain results
|
||||
And I stop the M2 MCP stub server
|
||||
|
||||
Scenario: MCP stub server lifecycle guards
|
||||
Given I have an M2 MCP stub server that is stopped
|
||||
When I try to discover tools from the stopped M2 stub
|
||||
Then an M2 RuntimeError should be raised
|
||||
When I start and invoke M2 MCP stub tool "mcp/fetch" with url "http://example.com"
|
||||
Then the M2 MCP stub fetch result should have status 200
|
||||
And the M2 MCP stub invocation log should have 1 entries
|
||||
@@ -0,0 +1,194 @@
|
||||
"""MCP stub server mock for M2 actor + tool source smoke tests.
|
||||
|
||||
Simulates MCP tool discovery and invocation in-process without requiring
|
||||
a real MCP adapter. Returns deterministic tool descriptors and execution
|
||||
results for Behave-level testing.
|
||||
|
||||
This mock lives in ``features/mocks/`` per ADR-022 (no mocks in src/).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class McpStubTool:
|
||||
"""Descriptor for a single tool exposed by the MCP stub server."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
description: str,
|
||||
input_schema: dict[str, Any] | None = None,
|
||||
output_schema: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = input_schema or {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
}
|
||||
self.output_schema = output_schema or {
|
||||
"type": "object",
|
||||
"properties": {"result": {"type": "string"}},
|
||||
}
|
||||
|
||||
|
||||
class McpStubServer:
|
||||
"""In-process MCP stub server for deterministic testing.
|
||||
|
||||
Provides ``discover()`` and ``invoke()`` methods that mirror the
|
||||
future MCP adapter interface (#159) without requiring network I/O.
|
||||
|
||||
Usage::
|
||||
|
||||
server = McpStubServer()
|
||||
server.start()
|
||||
tools = server.discover()
|
||||
result = server.invoke("mcp/search", {"query": "hello"})
|
||||
server.stop()
|
||||
"""
|
||||
|
||||
#: Default tools returned by ``discover()``.
|
||||
DEFAULT_TOOLS: ClassVar[list[McpStubTool]] = [
|
||||
McpStubTool(
|
||||
name="mcp/search",
|
||||
description="Stub MCP search tool",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
output_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"results": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
},
|
||||
),
|
||||
McpStubTool(
|
||||
name="mcp/fetch",
|
||||
description="Stub MCP fetch tool",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"url": {"type": "string"}},
|
||||
"required": ["url"],
|
||||
},
|
||||
output_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string"},
|
||||
"status": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
),
|
||||
McpStubTool(
|
||||
name="mcp/transform",
|
||||
description="Stub MCP transform tool",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {"type": "string"},
|
||||
"format": {"type": "string"},
|
||||
},
|
||||
"required": ["data"],
|
||||
},
|
||||
output_schema={
|
||||
"type": "object",
|
||||
"properties": {"transformed": {"type": "string"}},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
def __init__(self, tools: list[McpStubTool] | None = None) -> None:
|
||||
self._tools = tools or list(self.DEFAULT_TOOLS)
|
||||
self._running = False
|
||||
self._invocation_log: list[dict[str, Any]] = []
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
"""Whether the stub server is currently started."""
|
||||
return self._running
|
||||
|
||||
@property
|
||||
def invocation_log(self) -> list[dict[str, Any]]:
|
||||
"""Log of all invocations since the last ``start()``."""
|
||||
return list(self._invocation_log)
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the stub server (in-process, no network)."""
|
||||
self._running = True
|
||||
self._invocation_log = []
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop the stub server."""
|
||||
self._running = False
|
||||
|
||||
def discover(self) -> list[McpStubTool]:
|
||||
"""Return the list of available tools.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the server is not running.
|
||||
"""
|
||||
if not self._running:
|
||||
msg = "McpStubServer is not running. Call start() first."
|
||||
raise RuntimeError(msg)
|
||||
return list(self._tools)
|
||||
|
||||
def invoke(self, tool_name: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Invoke a stub tool by name with deterministic results.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the server is not running.
|
||||
KeyError: If the tool name is not found.
|
||||
"""
|
||||
if not self._running:
|
||||
msg = "McpStubServer is not running. Call start() first."
|
||||
raise RuntimeError(msg)
|
||||
|
||||
tool = self._find_tool(tool_name)
|
||||
if tool is None:
|
||||
msg = f"Tool '{tool_name}' not found in MCP stub server."
|
||||
raise KeyError(msg)
|
||||
|
||||
# Generate deterministic result based on tool name
|
||||
result = self._generate_result(tool, params)
|
||||
self._invocation_log.append(
|
||||
{
|
||||
"tool_name": tool_name,
|
||||
"params": params,
|
||||
"result": result,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def _find_tool(self, name: str) -> McpStubTool | None:
|
||||
"""Find a tool by name."""
|
||||
for tool in self._tools:
|
||||
if tool.name == name:
|
||||
return tool
|
||||
return None
|
||||
|
||||
def _generate_result(
|
||||
self, tool: McpStubTool, params: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Generate a deterministic result for a tool invocation."""
|
||||
if tool.name == "mcp/search":
|
||||
query = params.get("query", "")
|
||||
return {
|
||||
"results": [f"result_1_for_{query}", f"result_2_for_{query}"],
|
||||
}
|
||||
if tool.name == "mcp/fetch":
|
||||
url = params.get("url", "")
|
||||
return {
|
||||
"content": f"<html>stub content for {url}</html>",
|
||||
"status": 200,
|
||||
}
|
||||
if tool.name == "mcp/transform":
|
||||
data = params.get("data", "")
|
||||
fmt = params.get("format", "upper")
|
||||
transformed = data.upper() if fmt == "upper" else data.lower()
|
||||
return {"transformed": transformed}
|
||||
|
||||
# Generic fallback
|
||||
return {"result": f"stub result from {tool.name}"}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -621,6 +622,188 @@ def step_validate_actor_rejects(context: Context, actor: str) -> None:
|
||||
pass # Expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Automation profile resolution (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the cli extensions error output mentions available profiles")
|
||||
def step_error_mentions_available(context: Context) -> None:
|
||||
"""Verify error output lists available automation profiles."""
|
||||
output = context.last_result.output
|
||||
# The error message should mention at least some builtin profiles
|
||||
assert "Available" in output or "manual" in output or "trusted" in output, (
|
||||
f"Expected available profiles in error output: {output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Invariant ordering (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I run plan use with three invariants "{inv1}" and "{inv2}" and "{inv3}"')
|
||||
def step_run_plan_use_three_invariants(
|
||||
context: Context, inv1: str, inv2: str, inv3: str
|
||||
) -> None:
|
||||
"""Run plan use with three --invariant flags."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
inv1,
|
||||
"--invariant",
|
||||
inv2,
|
||||
"--invariant",
|
||||
inv3,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then(
|
||||
"the cli extensions service received invariants in order "
|
||||
'"{inv1}" then "{inv2}" then "{inv3}"'
|
||||
)
|
||||
def step_service_received_ordered_invariants(
|
||||
context: Context, inv1: str, inv2: str, inv3: str
|
||||
) -> None:
|
||||
"""Verify invariants were passed in exact insertion order."""
|
||||
call_args = context.mock_service.use_action.call_args
|
||||
assert call_args is not None, "use_action was not called"
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
assert invariants is not None, "No invariants passed to use_action"
|
||||
texts = [inv.text for inv in invariants]
|
||||
assert len(texts) >= 3, f"Expected at least 3 invariants, got {len(texts)}"
|
||||
assert texts[0] == inv1, f"Expected '{inv1}' at index 0, got '{texts[0]}'"
|
||||
assert texts[1] == inv2, f"Expected '{inv2}' at index 1, got '{texts[1]}'"
|
||||
assert texts[2] == inv3, f"Expected '{inv3}' at index 2, got '{texts[2]}'"
|
||||
|
||||
|
||||
@when(
|
||||
"I run plan use with five invariants "
|
||||
'"{inv1}" and "{inv2}" and "{inv3}" and "{inv4}" and "{inv5}"'
|
||||
)
|
||||
def step_run_plan_use_five_invariants(
|
||||
context: Context,
|
||||
inv1: str,
|
||||
inv2: str,
|
||||
inv3: str,
|
||||
inv4: str,
|
||||
inv5: str,
|
||||
) -> None:
|
||||
"""Run plan use with five --invariant flags."""
|
||||
result = context.runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
inv1,
|
||||
"--invariant",
|
||||
inv2,
|
||||
"--invariant",
|
||||
inv3,
|
||||
"--invariant",
|
||||
inv4,
|
||||
"--invariant",
|
||||
inv5,
|
||||
],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then(
|
||||
"the cli extensions service received five invariants in exact order "
|
||||
'"{inv1}" "{inv2}" "{inv3}" "{inv4}" "{inv5}"'
|
||||
)
|
||||
def step_service_received_five_ordered(
|
||||
context: Context,
|
||||
inv1: str,
|
||||
inv2: str,
|
||||
inv3: str,
|
||||
inv4: str,
|
||||
inv5: str,
|
||||
) -> None:
|
||||
"""Verify five invariants in exact insertion order."""
|
||||
call_args = context.mock_service.use_action.call_args
|
||||
assert call_args is not None, "use_action was not called"
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
assert invariants is not None, "No invariants passed"
|
||||
texts = [inv.text for inv in invariants]
|
||||
expected = [inv1, inv2, inv3, inv4, inv5]
|
||||
assert texts == expected, f"Expected {expected}, got {texts}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DEEP COVERAGE: Output snapshot assertions (#326)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the cli extensions output is valid JSON")
|
||||
def step_output_is_valid_json(context: Context) -> None:
|
||||
"""Verify last result output is valid JSON."""
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Exit code: {context.last_result.exit_code}, "
|
||||
f"output: {context.last_result.output}"
|
||||
)
|
||||
output = context.last_result.output.strip()
|
||||
try:
|
||||
context.parsed_json = json.loads(output)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(
|
||||
f"Output is not valid JSON: {exc}\nOutput: {output}"
|
||||
) from exc
|
||||
|
||||
|
||||
@then('the cli extensions json output contains key "{key}"')
|
||||
def step_json_output_contains_key(context: Context, key: str) -> None:
|
||||
"""Verify parsed JSON contains a specific key."""
|
||||
parsed = getattr(context, "parsed_json", None)
|
||||
if parsed is None:
|
||||
# Parse if not already parsed
|
||||
output = context.last_result.output.strip()
|
||||
parsed = json.loads(output)
|
||||
# Handle both dict and list-of-dict cases
|
||||
if isinstance(parsed, list):
|
||||
assert any(key in item for item in parsed if isinstance(item, dict)), (
|
||||
f"Key '{key}' not found in any list item"
|
||||
)
|
||||
else:
|
||||
assert key in parsed, f"Key '{key}' not in JSON: {list(parsed.keys())}"
|
||||
|
||||
|
||||
@then('the cli extensions output contains yaml field "{field}"')
|
||||
def step_output_contains_yaml_field(context: Context, field: str) -> None:
|
||||
"""Verify output contains a YAML field (key: value pattern)."""
|
||||
assert context.last_result.exit_code == 0
|
||||
output = context.last_result.output
|
||||
assert field in output, f"Expected YAML field '{field}' in output: {output}"
|
||||
|
||||
|
||||
@then("the cli extensions json invariants list has length {length:d}")
|
||||
def step_json_invariants_length(context: Context, length: int) -> None:
|
||||
"""Verify JSON invariants list has expected length."""
|
||||
parsed = getattr(context, "parsed_json", None)
|
||||
if parsed is None:
|
||||
output = context.last_result.output.strip()
|
||||
parsed = json.loads(output)
|
||||
invariants = parsed.get("invariants", [])
|
||||
assert len(invariants) == length, (
|
||||
f"Expected {length} invariants, got {len(invariants)}: {invariants}"
|
||||
)
|
||||
|
||||
|
||||
@then('the cli extensions json output string contains "{text}"')
|
||||
def step_json_output_string_contains(context: Context, text: str) -> None:
|
||||
"""Verify the raw JSON output string contains the given text."""
|
||||
assert context.last_result.exit_code == 0
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in JSON output: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
"""Step definitions for M1 source-code plan lifecycle smoke tests.
|
||||
|
||||
All step names are prefixed with ``m1 smoke`` to avoid ``AmbiguousStep``
|
||||
conflicts with existing steps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import yaml
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app
|
||||
from cleveragents.cli.commands.plan import app as plan_app
|
||||
from cleveragents.cli.commands.project import app as project_app
|
||||
from cleveragents.domain.models.core.action import Action, ActionState
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "fixtures" / "m1"
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
|
||||
|
||||
def _make_m1_plan(
|
||||
*,
|
||||
name: str = "local/m1-smoke-plan",
|
||||
action_name: str = "local/m1-source-review",
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
arguments: dict[str, object] | None = None,
|
||||
is_terminal: bool = False,
|
||||
) -> Plan:
|
||||
"""Create a Plan instance for M1 smoke tests."""
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="M1 smoke test plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name=action_name,
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments=dict(arguments) if arguments else {},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
def _make_m1_action(
|
||||
name: str = "local/m1-source-review",
|
||||
) -> Action:
|
||||
"""Create an Action for M1 smoke tests."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke test runner")
|
||||
def step_m1_smoke_runner(context: Context) -> None:
|
||||
"""Set up the CLI runner for M1 smoke tests."""
|
||||
context.runner = CliRunner()
|
||||
|
||||
|
||||
@given("a m1 smoke mocked lifecycle service")
|
||||
def step_m1_smoke_mock_service(context: Context) -> None:
|
||||
"""Set up the mocked lifecycle service for M1 smoke tests."""
|
||||
from cleveragents.application.services.plan_lifecycle_service import (
|
||||
ActionNotAvailableError,
|
||||
)
|
||||
|
||||
context.mock_service = MagicMock()
|
||||
# Default: get_action_by_name raises for any action not explicitly set up
|
||||
context._m1_known_actions = {} # type-checked in step helpers
|
||||
|
||||
def _get_action_by_name_side_effect(name: str) -> Action:
|
||||
if name in context._m1_known_actions:
|
||||
return context._m1_known_actions[name]
|
||||
raise ActionNotAvailableError(action_name=name, state=ActionState.ARCHIVED)
|
||||
|
||||
context.mock_service.get_action_by_name.side_effect = (
|
||||
_get_action_by_name_side_effect
|
||||
)
|
||||
|
||||
context.plan_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=context.mock_service,
|
||||
)
|
||||
context.action_patcher = patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=context.mock_service,
|
||||
)
|
||||
context.project_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_namespaced_project_repo",
|
||||
)
|
||||
context.resource_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_resource_registry_service",
|
||||
)
|
||||
context.link_patcher = patch(
|
||||
"cleveragents.cli.commands.project._get_resource_link_repo",
|
||||
)
|
||||
context.plan_patcher.start()
|
||||
context.action_patcher.start()
|
||||
context.mock_project_repo = context.project_patcher.start()
|
||||
context.mock_resource_svc = context.resource_patcher.start()
|
||||
context.mock_link_repo = context.link_patcher.start()
|
||||
context.last_result = None
|
||||
context.captured_plan_id = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixture loading steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I m1 smoke load the git repo fixture")
|
||||
def step_m1_load_git_repo(context: Context) -> None:
|
||||
"""Load the git repo fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "git_repo.json"
|
||||
with open(fixture_path) as f:
|
||||
context.git_repo_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m1 smoke git repo fixture should have a minimal repo entry")
|
||||
def step_m1_git_repo_has_minimal(context: Context) -> None:
|
||||
"""Verify fixture has a minimal_git_repo entry."""
|
||||
fixtures = context.git_repo_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "minimal_git_repo" in names, f"Expected 'minimal_git_repo' in {names}"
|
||||
|
||||
|
||||
@then("the m1 smoke minimal repo should contain expected files")
|
||||
def step_m1_git_repo_has_files(context: Context) -> None:
|
||||
"""Verify the minimal repo defines expected files."""
|
||||
fixtures = context.git_repo_fixtures["fixtures"]
|
||||
minimal = next(f for f in fixtures if f["name"] == "minimal_git_repo")
|
||||
files = minimal["files"]
|
||||
assert "README.md" in files, "Expected README.md in fixture files"
|
||||
assert "src/main.py" in files, "Expected src/main.py in fixture files"
|
||||
|
||||
|
||||
@when("I m1 smoke load the git checkout resource fixture")
|
||||
def step_m1_load_git_checkout(context: Context) -> None:
|
||||
"""Load the git-checkout resource fixture JSON."""
|
||||
fixture_path = _FIXTURES_DIR / "git_checkout_resource.json"
|
||||
with open(fixture_path) as f:
|
||||
context.checkout_fixtures = json.load(f)
|
||||
|
||||
|
||||
@then("the m1 smoke git checkout fixture should have a minimal entry")
|
||||
def step_m1_checkout_has_minimal(context: Context) -> None:
|
||||
"""Verify fixture has a minimal_git_checkout entry."""
|
||||
fixtures = context.checkout_fixtures["fixtures"]
|
||||
names = [f["name"] for f in fixtures]
|
||||
assert "minimal_git_checkout" in names, (
|
||||
f"Expected 'minimal_git_checkout' in {names}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke minimal checkout should have type "{rtype}"')
|
||||
def step_m1_checkout_type(context: Context, rtype: str) -> None:
|
||||
"""Verify the minimal checkout has the expected resource type."""
|
||||
fixtures = context.checkout_fixtures["fixtures"]
|
||||
minimal = next(f for f in fixtures if f["name"] == "minimal_git_checkout")
|
||||
assert minimal["resource_type"] == rtype, (
|
||||
f"Expected type '{rtype}', got '{minimal['resource_type']}'"
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke load the action YAML fixture")
|
||||
def step_m1_load_action_yaml(context: Context) -> None:
|
||||
"""Load the action YAML fixture."""
|
||||
fixture_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
with open(fixture_path) as f:
|
||||
context.action_fixture = yaml.safe_load(f)
|
||||
|
||||
|
||||
@then('the m1 smoke action fixture should define strategy actor "{actor}"')
|
||||
def step_m1_action_strategy_actor(context: Context, actor: str) -> None:
|
||||
"""Verify the action fixture defines the expected strategy actor."""
|
||||
assert context.action_fixture["strategy_actor"] == actor
|
||||
|
||||
|
||||
@then('the m1 smoke action fixture should define execution actor "{actor}"')
|
||||
def step_m1_action_execution_actor(context: Context, actor: str) -> None:
|
||||
"""Verify the action fixture defines the expected execution actor."""
|
||||
assert context.action_fixture["execution_actor"] == actor
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action create steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke temporary action config file")
|
||||
def step_m1_temp_action_config(context: Context) -> None:
|
||||
"""Create a temp YAML config file for action create."""
|
||||
fixture_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
context.m1_action_config = str(fixture_path)
|
||||
context.mock_service.create_action.return_value = _make_m1_action()
|
||||
|
||||
|
||||
@when("I m1 smoke invoke action create with the config")
|
||||
def step_m1_invoke_action_create(context: Context) -> None:
|
||||
"""Invoke action create CLI with config file."""
|
||||
result = context.runner.invoke(
|
||||
action_app,
|
||||
["create", "--config", context.m1_action_config],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke action create should succeed")
|
||||
def step_m1_action_create_ok(context: Context) -> None:
|
||||
"""Verify action create succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke action output should contain "{text}"')
|
||||
def step_m1_action_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify action output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project + resource link steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when('I m1 smoke create a temp project "{name}"')
|
||||
def step_m1_create_project(context: Context, name: str) -> None:
|
||||
"""Create a temp project via the mock."""
|
||||
mock_repo = context.mock_project_repo.return_value
|
||||
mock_project = MagicMock()
|
||||
mock_project.namespaced_name = name
|
||||
mock_project.namespace = name.split("/")[0]
|
||||
mock_project.name = name.split("/")[1]
|
||||
mock_project.description = "M1 smoke test project"
|
||||
mock_project.linked_resources = []
|
||||
mock_project.created_at = datetime.now()
|
||||
mock_project.updated_at = datetime.now()
|
||||
mock_repo.create.return_value = None
|
||||
mock_repo.get.return_value = mock_project
|
||||
result = context.runner.invoke(
|
||||
project_app,
|
||||
["create", name, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke project creation should succeed")
|
||||
def step_m1_project_create_ok(context: Context) -> None:
|
||||
"""Verify project creation succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then('the m1 smoke project output should contain "{text}"')
|
||||
def step_m1_project_output_contains(context: Context, text: str) -> None:
|
||||
"""Verify project output contains expected text."""
|
||||
output = context.last_result.output
|
||||
assert text in output, f"Expected '{text}' in: {output}"
|
||||
|
||||
|
||||
@given('a m1 smoke project "{name}" exists')
|
||||
def step_m1_project_exists(context: Context, name: str) -> None:
|
||||
"""Set up existing project mock."""
|
||||
mock_repo = context.mock_project_repo.return_value
|
||||
mock_project = MagicMock()
|
||||
mock_project.namespaced_name = name
|
||||
mock_project.namespace = name.split("/")[0]
|
||||
mock_project.name = name.split("/")[1]
|
||||
mock_project.linked_resources = []
|
||||
mock_repo.get.return_value = mock_project
|
||||
|
||||
|
||||
@given('a m1 smoke resource "{name}" exists')
|
||||
def step_m1_resource_exists(context: Context, name: str) -> None:
|
||||
"""Set up existing resource mock."""
|
||||
mock_registry = context.mock_resource_svc.return_value
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_id = "01M1RESOURCE000000000000001"
|
||||
mock_resource.name = name
|
||||
mock_registry.show_resource.return_value = mock_resource
|
||||
|
||||
|
||||
@when('I m1 smoke link resource "{resource}" to project "{project}"')
|
||||
def step_m1_link_resource(context: Context, resource: str, project: str) -> None:
|
||||
"""Link a resource to a project via CLI."""
|
||||
mock_link_repo = context.mock_link_repo.return_value
|
||||
mock_link_repo.create_link.return_value = None
|
||||
result = context.runner.invoke(
|
||||
project_app,
|
||||
["link-resource", project, resource, "--format", "plain"],
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke link should succeed")
|
||||
def step_m1_link_ok(context: Context) -> None:
|
||||
"""Verify resource link succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan use steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given('a m1 smoke action "{name}" exists')
|
||||
def step_m1_action_exists(context: Context, name: str) -> None:
|
||||
"""Set up an existing action for plan use."""
|
||||
action = _make_m1_action(name)
|
||||
context._m1_known_actions[name] = action
|
||||
context.mock_service.use_action.return_value = _make_m1_plan(action_name=name)
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use with action "{name}"')
|
||||
def step_m1_plan_use(context: Context, name: str) -> None:
|
||||
"""Invoke plan use with the given action."""
|
||||
result = context.runner.invoke(plan_app, ["use", name])
|
||||
context.last_result = result
|
||||
if result.exit_code == 0:
|
||||
context.captured_plan_id = _PLAN_ULID
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use linking project "{project}" to action "{name}"')
|
||||
def step_m1_plan_use_with_project(context: Context, project: str, name: str) -> None:
|
||||
"""Invoke plan use with action and project."""
|
||||
result = context.runner.invoke(plan_app, ["use", name, project])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use passing arg "{arg_str}" to action "{name}"')
|
||||
def step_m1_plan_use_with_arg(context: Context, arg_str: str, name: str) -> None:
|
||||
"""Invoke plan use with action and --arg."""
|
||||
result = context.runner.invoke(plan_app, ["use", name, "--arg", arg_str])
|
||||
context.last_result = result
|
||||
if result.exit_code == 0:
|
||||
context.captured_plan_id = _PLAN_ULID
|
||||
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@when('I m1 smoke invoke plan use with invalid strategy actor "{actor}"')
|
||||
def step_m1_plan_use_invalid_actor(context: Context, actor: str) -> None:
|
||||
"""Invoke plan use with an invalid actor format."""
|
||||
result = context.runner.invoke(
|
||||
plan_app, ["use", "local/m1-source-review", "--strategy-actor", actor]
|
||||
)
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan use should succeed")
|
||||
def step_m1_plan_use_ok(context: Context) -> None:
|
||||
"""Verify plan use succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan use should fail")
|
||||
def step_m1_plan_use_fail(context: Context) -> None:
|
||||
"""Verify plan use failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
@then('the m1 smoke plan should be in phase "{phase}"')
|
||||
def step_m1_plan_phase(context: Context, phase: str) -> None:
|
||||
"""Verify plan is in the expected phase."""
|
||||
output = context.last_result.output
|
||||
assert phase in output.lower(), f"Expected phase '{phase}' in output: {output}"
|
||||
|
||||
|
||||
@then("the m1 smoke captured plan id should not be empty")
|
||||
def step_m1_plan_id_captured(context: Context) -> None:
|
||||
"""Verify we captured a plan ID."""
|
||||
assert context.captured_plan_id is not None
|
||||
assert len(context.captured_plan_id) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan execute steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in strategize phase")
|
||||
def step_m1_plan_in_strategize(context: Context) -> None:
|
||||
"""Set up a plan in strategize/complete phase for execute."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.STRATEGIZE,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
context.mock_service.execute_plan.return_value = _make_m1_plan(
|
||||
phase=PlanPhase.EXECUTE,
|
||||
state=ProcessingState.QUEUED,
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan execute")
|
||||
def step_m1_plan_execute(context: Context) -> None:
|
||||
"""Invoke plan execute."""
|
||||
result = context.runner.invoke(plan_app, ["execute"])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan execute should succeed")
|
||||
def step_m1_plan_execute_ok(context: Context) -> None:
|
||||
"""Verify plan execute succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan execute should fail")
|
||||
def step_m1_plan_execute_fail(context: Context) -> None:
|
||||
"""Verify plan execute failed."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code != 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan diff steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in execute phase with changeset")
|
||||
def step_m1_plan_with_changeset(context: Context) -> None:
|
||||
"""Set up a plan in execute phase with a changeset mock."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.EXECUTE,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.get_plan.return_value = plan
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
# Mock the apply service since plan diff uses _get_apply_service()
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = "No changes detected."
|
||||
context.apply_patcher = patch(
|
||||
"cleveragents.cli.commands.plan._get_apply_service",
|
||||
return_value=mock_apply_svc,
|
||||
)
|
||||
context.apply_patcher.start()
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan diff")
|
||||
def step_m1_plan_diff(context: Context) -> None:
|
||||
"""Invoke plan diff."""
|
||||
result = context.runner.invoke(plan_app, ["diff", _PLAN_ULID])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan diff should succeed")
|
||||
def step_m1_plan_diff_ok(context: Context) -> None:
|
||||
"""Verify plan diff succeeded."""
|
||||
assert context.last_result is not None
|
||||
# diff may succeed or return 0 even without changes
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan apply steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("a m1 smoke plan exists in apply phase")
|
||||
def step_m1_plan_in_apply(context: Context) -> None:
|
||||
"""Set up a plan in apply phase."""
|
||||
plan = _make_m1_plan(
|
||||
phase=PlanPhase.APPLY,
|
||||
state=ProcessingState.COMPLETE,
|
||||
)
|
||||
context.mock_service.list_plans.return_value = [plan]
|
||||
context.mock_service.get_plan.return_value = plan
|
||||
context.mock_service.apply_plan.return_value = _make_m1_plan(
|
||||
phase=PlanPhase.APPLY,
|
||||
state=ProcessingState.APPLIED,
|
||||
)
|
||||
# For execute attempts on an apply-phase plan, raise error
|
||||
from cleveragents.application.services.plan_lifecycle_service import (
|
||||
InvalidPhaseTransitionError,
|
||||
)
|
||||
|
||||
context.mock_service.execute_plan.side_effect = InvalidPhaseTransitionError(
|
||||
from_phase=PlanPhase.APPLY,
|
||||
to_phase=PlanPhase.EXECUTE,
|
||||
message="Cannot execute a plan in apply phase",
|
||||
)
|
||||
|
||||
|
||||
@when("I m1 smoke invoke plan lifecycle-apply")
|
||||
def step_m1_plan_apply(context: Context) -> None:
|
||||
"""Invoke plan lifecycle-apply."""
|
||||
result = context.runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
context.last_result = result
|
||||
|
||||
|
||||
@then("the m1 smoke plan apply should succeed")
|
||||
def step_m1_plan_apply_ok(context: Context) -> None:
|
||||
"""Verify plan apply succeeded."""
|
||||
assert context.last_result is not None
|
||||
assert context.last_result.exit_code == 0, (
|
||||
f"Expected exit 0, got {context.last_result.exit_code}. "
|
||||
f"Output: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
@then("the m1 smoke plan should be in terminal state")
|
||||
def step_m1_plan_terminal(context: Context) -> None:
|
||||
"""Verify the plan is in terminal state."""
|
||||
output = context.last_result.output.lower()
|
||||
assert "appl" in output or "terminal" in output, (
|
||||
f"Expected terminal state indicator in: {context.last_result.output}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def after_scenario(context: Context, scenario: object) -> None:
|
||||
"""Clean up patchers after each scenario."""
|
||||
for name in (
|
||||
"plan_patcher",
|
||||
"action_patcher",
|
||||
"project_patcher",
|
||||
"resource_patcher",
|
||||
"link_patcher",
|
||||
"apply_patcher",
|
||||
):
|
||||
patcher = getattr(context, name, None)
|
||||
if patcher:
|
||||
patcher.stop()
|
||||
@@ -0,0 +1,520 @@
|
||||
"""Step definitions for m2_actor_tool_smoke.feature.
|
||||
|
||||
All step names are prefixed with 'M2' to avoid AmbiguousStep conflicts
|
||||
with existing actor/skill/tool step files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.actor.loader import ActorLoader
|
||||
from cleveragents.actor.schema import ActorConfigSchema
|
||||
from cleveragents.domain.models.core.skill import (
|
||||
Skill,
|
||||
SkillInlineTool,
|
||||
)
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
)
|
||||
from cleveragents.skills.context import SkillContext
|
||||
from cleveragents.skills.protocol import SkillDefinition, SkillMetadata
|
||||
from cleveragents.skills.registry import SkillRegistry
|
||||
from cleveragents.skills.schema import SkillConfigSchema
|
||||
from cleveragents.tool.context import ToolExecutionContext
|
||||
from cleveragents.tool.lifecycle import (
|
||||
ToolDescriptor,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
)
|
||||
from cleveragents.tool.registry import ToolRegistry
|
||||
from cleveragents.tool.runtime import ToolSpec
|
||||
|
||||
# ── Fixture paths ──────────────────────────────────────────────────
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" / "m2"
|
||||
_ACTORS_DIR = _FIXTURES_DIR / "actors"
|
||||
_HIERARCHICAL_ACTOR = _ACTORS_DIR / "m2_hierarchical_actor.yaml"
|
||||
_SKILL_PACK = _FIXTURES_DIR / "m2_skill_pack.yaml"
|
||||
|
||||
|
||||
# ── Mock ToolInstance for lifecycle tests ──────────────────────────
|
||||
|
||||
|
||||
class _M2MockToolInstance:
|
||||
"""Minimal ToolInstance for M2 lifecycle testing."""
|
||||
|
||||
def __init__(self, name: str) -> None:
|
||||
self._name = name
|
||||
self._activated = False
|
||||
self._deactivated = False
|
||||
self._execute_count = 0
|
||||
|
||||
def discover(self) -> ToolDescriptor:
|
||||
return ToolDescriptor(
|
||||
name=self._name,
|
||||
description=f"M2 mock tool {self._name}",
|
||||
source="custom",
|
||||
)
|
||||
|
||||
def activate(self, ctx: ToolExecutionContext) -> None:
|
||||
self._activated = True
|
||||
|
||||
def execute(self, params: dict[str, Any], ctx: ToolExecutionContext) -> ToolResult:
|
||||
self._execute_count += 1
|
||||
return ToolResult(success=True, data={"echo": params.get("text", "")})
|
||||
|
||||
def deactivate(self, ctx: ToolExecutionContext) -> None:
|
||||
self._deactivated = True
|
||||
|
||||
@property
|
||||
def is_activated(self) -> bool:
|
||||
return self._activated
|
||||
|
||||
@property
|
||||
def is_deactivated(self) -> bool:
|
||||
return self._deactivated
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Actor YAML loading
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@given("I load the M2 hierarchical actor fixture")
|
||||
def step_m2_load_hierarchical_actor(context: Context) -> None:
|
||||
context._m2_actor = ActorConfigSchema.from_yaml_file(_HIERARCHICAL_ACTOR)
|
||||
|
||||
|
||||
@then('the M2 actor name should be "{name}"')
|
||||
def step_m2_actor_name(context: Context, name: str) -> None:
|
||||
actor: ActorConfigSchema = context._m2_actor
|
||||
assert actor.name == name, f"Expected '{name}', got '{actor.name}'"
|
||||
|
||||
|
||||
@then('the M2 actor type should be "{actor_type}"')
|
||||
def step_m2_actor_type(context: Context, actor_type: str) -> None:
|
||||
actor: ActorConfigSchema = context._m2_actor
|
||||
assert actor.type.value == actor_type, (
|
||||
f"Expected type '{actor_type}', got '{actor.type.value}'"
|
||||
)
|
||||
|
||||
|
||||
@then("the M2 actor should have {count:d} graph nodes")
|
||||
def step_m2_actor_node_count(context: Context, count: int) -> None:
|
||||
actor: ActorConfigSchema = context._m2_actor
|
||||
assert actor.route is not None, "Expected a route definition"
|
||||
assert len(actor.route.nodes) == count, (
|
||||
f"Expected {count} nodes, got {len(actor.route.nodes)}"
|
||||
)
|
||||
|
||||
|
||||
@then('the M2 actor entry node should be "{entry}"')
|
||||
def step_m2_actor_entry_node(context: Context, entry: str) -> None:
|
||||
actor: ActorConfigSchema = context._m2_actor
|
||||
assert actor.route is not None
|
||||
assert actor.route.entry_node == entry, (
|
||||
f"Expected entry '{entry}', got '{actor.route.entry_node}'"
|
||||
)
|
||||
|
||||
|
||||
@then('the M2 actor exit nodes should include "{exit_node}"')
|
||||
def step_m2_actor_exit_nodes(context: Context, exit_node: str) -> None:
|
||||
actor: ActorConfigSchema = context._m2_actor
|
||||
assert actor.route is not None
|
||||
assert exit_node in actor.route.exit_nodes, (
|
||||
f"Expected '{exit_node}' in exit nodes: {actor.route.exit_nodes}"
|
||||
)
|
||||
|
||||
|
||||
# ── Actor loader discovery from fixture dir ───────────────────────
|
||||
|
||||
|
||||
@given("I create an M2 actor loader from the fixture directory")
|
||||
def step_m2_create_loader(context: Context) -> None:
|
||||
context._m2_loader = ActorLoader(search_roots=[_ACTORS_DIR])
|
||||
|
||||
|
||||
@when("I run M2 actor discovery")
|
||||
def step_m2_run_discovery(context: Context) -> None:
|
||||
loader: ActorLoader = context._m2_loader
|
||||
context._m2_discovered = loader.discover()
|
||||
|
||||
|
||||
@then("the M2 loader should find {count:d} actors")
|
||||
def step_m2_loader_count(context: Context, count: int) -> None:
|
||||
loader: ActorLoader = context._m2_loader
|
||||
actors = loader.list_actors()
|
||||
assert len(actors) == count, (
|
||||
f"Expected {count} actors, got {len(actors)}: {[a.name for a in actors]}"
|
||||
)
|
||||
|
||||
|
||||
@then('the M2 loader should contain actor "{name}"')
|
||||
def step_m2_loader_contains(context: Context, name: str) -> None:
|
||||
loader: ActorLoader = context._m2_loader
|
||||
config = loader.get(name)
|
||||
assert config is not None, f"Actor '{name}' not found in M2 loader"
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Skill pack loading and registry
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@given("I load the M2 skill pack fixture")
|
||||
def step_m2_load_skill_pack(context: Context) -> None:
|
||||
context._m2_skill_config = SkillConfigSchema.from_yaml_file(_SKILL_PACK)
|
||||
|
||||
|
||||
@then('the M2 skill name should be "{name}"')
|
||||
def step_m2_skill_name(context: Context, name: str) -> None:
|
||||
cfg: SkillConfigSchema = context._m2_skill_config
|
||||
assert cfg.name == name, f"Expected '{name}', got '{cfg.name}'"
|
||||
|
||||
|
||||
@then("the M2 skill should have {count:d} tool references")
|
||||
def step_m2_skill_tool_refs(context: Context, count: int) -> None:
|
||||
cfg: SkillConfigSchema = context._m2_skill_config
|
||||
assert len(cfg.tools) == count, f"Expected {count} tool refs, got {len(cfg.tools)}"
|
||||
|
||||
|
||||
@then("the M2 skill should have {count:d} inline tools")
|
||||
def step_m2_skill_inline_tools(context: Context, count: int) -> None:
|
||||
cfg: SkillConfigSchema = context._m2_skill_config
|
||||
assert len(cfg.inline_tools) == count, (
|
||||
f"Expected {count} inline tools, got {len(cfg.inline_tools)}"
|
||||
)
|
||||
|
||||
|
||||
def _build_m2_skill_definition() -> SkillDefinition:
|
||||
"""Build a SkillDefinition from the M2 fixture for registry tests."""
|
||||
skill = Skill(
|
||||
name="m2test/file-ops-pack",
|
||||
description="M2 smoke test skill pack with file operation tools",
|
||||
tool_refs=["builtin/read_file", "builtin/list_files"],
|
||||
anonymous_tools=[
|
||||
SkillInlineTool(
|
||||
description="Echo input back for testing",
|
||||
source=ToolSource.CUSTOM,
|
||||
code="def m2_echo(text: str) -> str:\n return f'echo: {text}'",
|
||||
timeout=300,
|
||||
),
|
||||
],
|
||||
)
|
||||
metadata = SkillMetadata.from_skill(skill)
|
||||
return SkillDefinition(
|
||||
skill=skill,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
@given("I register the M2 skill pack in a skill registry")
|
||||
def step_m2_register_skill_pack(context: Context) -> None:
|
||||
registry = SkillRegistry()
|
||||
defn = _build_m2_skill_definition()
|
||||
registry.register(defn)
|
||||
context._m2_skill_registry = registry
|
||||
|
||||
|
||||
@when('I resolve tools for M2 skill "{name}"')
|
||||
def step_m2_resolve_tools(context: Context, name: str) -> None:
|
||||
registry: SkillRegistry = context._m2_skill_registry
|
||||
context._m2_resolved_tools = registry.resolve_tools(name)
|
||||
|
||||
|
||||
@then("the M2 resolved tools should include a tool_ref entry")
|
||||
def step_m2_resolved_has_tool_ref(context: Context) -> None:
|
||||
entries = context._m2_resolved_tools
|
||||
has_ref = any(not e.is_inline for e in entries)
|
||||
assert has_ref, "Expected at least one tool_ref entry in resolved tools"
|
||||
|
||||
|
||||
@then("the M2 resolved tools should include an inline entry")
|
||||
def step_m2_resolved_has_inline(context: Context) -> None:
|
||||
entries = context._m2_resolved_tools
|
||||
has_inline = any(e.is_inline for e in entries)
|
||||
assert has_inline, "Expected at least one inline entry in resolved tools"
|
||||
|
||||
|
||||
@when("I list all M2 skills in the registry")
|
||||
def step_m2_list_skills(context: Context) -> None:
|
||||
registry: SkillRegistry = context._m2_skill_registry
|
||||
context._m2_skill_list = registry.list_all()
|
||||
|
||||
|
||||
@then('the M2 skill list should contain "{name}"')
|
||||
def step_m2_skill_list_contains(context: Context, name: str) -> None:
|
||||
skill_list = context._m2_skill_list
|
||||
names = [m.name for m in skill_list]
|
||||
assert name in names, f"Expected '{name}' in {names}"
|
||||
|
||||
|
||||
@then("the M2 skill list should have {count:d} entries")
|
||||
def step_m2_skill_list_count(context: Context, count: int) -> None:
|
||||
skill_list = context._m2_skill_list
|
||||
assert len(skill_list) == count, f"Expected {count} entries, got {len(skill_list)}"
|
||||
|
||||
|
||||
# ── Skill context invocation ──────────────────────────────────────
|
||||
|
||||
|
||||
@given("I create an M2 skill context with sandbox")
|
||||
def step_m2_create_skill_context(context: Context) -> None:
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_skill_ctx_"))
|
||||
context._m2_skill_ctx = SkillContext(
|
||||
plan_id="m2-plan-001",
|
||||
project_id="m2-project-001",
|
||||
sandbox_path=tmp,
|
||||
)
|
||||
|
||||
|
||||
@when('I register a tool invocation for "{tool_name}" in the M2 context')
|
||||
def step_m2_register_invocation(context: Context, tool_name: str) -> None:
|
||||
ctx: SkillContext = context._m2_skill_ctx
|
||||
ctx.register_tool_invocation(
|
||||
tool_name=tool_name,
|
||||
input_data={"path": "test.txt"},
|
||||
output_data={"content": "hello"},
|
||||
duration_ms=1.5,
|
||||
)
|
||||
|
||||
|
||||
@then("the M2 skill context should have {count:d} invocation records")
|
||||
def step_m2_skill_ctx_invocations(context: Context, count: int) -> None:
|
||||
ctx: SkillContext = context._m2_skill_ctx
|
||||
assert len(ctx.change_tracker) == count, (
|
||||
f"Expected {count} records, got {len(ctx.change_tracker)}"
|
||||
)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# Tool lifecycle
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@given('I create an M2 tool runtime with a mock tool "{name}"')
|
||||
def step_m2_create_tool_runtime(context: Context, name: str) -> None:
|
||||
tool = Tool(
|
||||
name=name,
|
||||
description=f"M2 mock tool {name}",
|
||||
source=ToolSource.BUILTIN,
|
||||
tool_type=ToolType.TOOL,
|
||||
capability=ToolCapability(writes=False, read_only=True, checkpointable=False),
|
||||
timeout=300,
|
||||
)
|
||||
instance = _M2MockToolInstance(name)
|
||||
runtime = ToolRuntime()
|
||||
runtime.register_tool(tool, instance)
|
||||
context._m2_runtime = runtime
|
||||
context._m2_mock_instance = instance
|
||||
|
||||
|
||||
@given('I create an M2 execution context with plan "{plan_id}"')
|
||||
def step_m2_create_exec_ctx(context: Context, plan_id: str) -> None:
|
||||
context._m2_exec_ctx = ToolExecutionContext(plan_id=plan_id)
|
||||
|
||||
|
||||
@when('I discover M2 tool "{name}"')
|
||||
def step_m2_discover_tool(context: Context, name: str) -> None:
|
||||
runtime: ToolRuntime = context._m2_runtime
|
||||
context._m2_descriptor = runtime.discover(name)
|
||||
|
||||
|
||||
@then('the M2 discovered descriptor name should be "{name}"')
|
||||
def step_m2_descriptor_name(context: Context, name: str) -> None:
|
||||
desc: ToolDescriptor = context._m2_descriptor
|
||||
assert desc.name == name, f"Expected '{name}', got '{desc.name}'"
|
||||
|
||||
|
||||
@when('I activate M2 tool "{name}"')
|
||||
def step_m2_activate_tool(context: Context, name: str) -> None:
|
||||
runtime: ToolRuntime = context._m2_runtime
|
||||
ctx: ToolExecutionContext = context._m2_exec_ctx
|
||||
runtime.activate(name, ctx)
|
||||
|
||||
|
||||
@when('I execute M2 tool "{name}" with params text "{text}"')
|
||||
def step_m2_execute_tool(context: Context, name: str, text: str) -> None:
|
||||
runtime: ToolRuntime = context._m2_runtime
|
||||
ctx: ToolExecutionContext = context._m2_exec_ctx
|
||||
context._m2_exec_result = runtime.execute(name, {"text": text}, ctx)
|
||||
|
||||
|
||||
@then("the M2 execution result should be successful")
|
||||
def step_m2_exec_result_success(context: Context) -> None:
|
||||
result: ToolResult = context._m2_exec_result
|
||||
assert result.success, f"Expected success, got error: {result.error}"
|
||||
|
||||
|
||||
@when('I deactivate M2 tool "{name}"')
|
||||
def step_m2_deactivate_tool(context: Context, name: str) -> None:
|
||||
runtime: ToolRuntime = context._m2_runtime
|
||||
ctx: ToolExecutionContext = context._m2_exec_ctx
|
||||
runtime.deactivate(name, ctx)
|
||||
|
||||
|
||||
@then("the M2 tool should be deactivated")
|
||||
def step_m2_tool_deactivated(context: Context) -> None:
|
||||
instance: _M2MockToolInstance = context._m2_mock_instance
|
||||
assert instance.is_deactivated, "Expected mock tool to be deactivated"
|
||||
|
||||
|
||||
# ── Tool registry ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@given("I create an M2 tool registry")
|
||||
def step_m2_create_tool_registry(context: Context) -> None:
|
||||
context._m2_tool_registry = ToolRegistry()
|
||||
|
||||
|
||||
def _m2_noop_handler(**kwargs: Any) -> dict[str, bool]:
|
||||
"""Typed no-op handler for M2 tool registry test specs."""
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@when('I register an M2 tool spec "{name}"')
|
||||
def step_m2_register_tool_spec(context: Context, name: str) -> None:
|
||||
registry: ToolRegistry = context._m2_tool_registry
|
||||
spec = ToolSpec(
|
||||
name=name,
|
||||
description=f"M2 test tool {name}",
|
||||
handler=_m2_noop_handler,
|
||||
)
|
||||
registry.register(spec)
|
||||
|
||||
|
||||
@then('getting M2 tool "{name}" should return a spec')
|
||||
def step_m2_get_tool_spec(context: Context, name: str) -> None:
|
||||
registry: ToolRegistry = context._m2_tool_registry
|
||||
spec = registry.get(name)
|
||||
assert spec is not None, f"Expected spec for '{name}', got None"
|
||||
|
||||
|
||||
@then('listing M2 tools should include "{name}"')
|
||||
def step_m2_list_tools_include(context: Context, name: str) -> None:
|
||||
registry: ToolRegistry = context._m2_tool_registry
|
||||
specs = registry.list_tools()
|
||||
names = [s.name for s in specs]
|
||||
assert name in names, f"Expected '{name}' in {names}"
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
# MCP stub tool discovery and invocation
|
||||
# ════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def _get_mcp_stub_class() -> type:
|
||||
"""Lazily import McpStubServer to avoid top-level sys.path mutation."""
|
||||
import sys as _sys
|
||||
|
||||
_mocks_dir = str(Path(__file__).resolve().parent.parent / "mocks")
|
||||
if _mocks_dir not in _sys.path:
|
||||
_sys.path.insert(0, _mocks_dir)
|
||||
|
||||
from mcp_stub_server import McpStubServer
|
||||
|
||||
return McpStubServer
|
||||
|
||||
|
||||
@given("I start the M2 MCP stub server")
|
||||
def step_m2_start_mcp_stub(context: Context) -> None:
|
||||
cls = _get_mcp_stub_class()
|
||||
server = cls()
|
||||
server.start()
|
||||
context._m2_mcp_stub = server
|
||||
|
||||
|
||||
@when("I discover tools from the M2 MCP stub")
|
||||
def step_m2_discover_mcp_tools(context: Context) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
context._m2_mcp_tools = server.discover()
|
||||
|
||||
|
||||
@then("the M2 stub should expose {count:d} tools")
|
||||
def step_m2_stub_tool_count(context: Context, count: int) -> None:
|
||||
tools = context._m2_mcp_tools
|
||||
assert len(tools) == count, f"Expected {count} tools, got {len(tools)}"
|
||||
|
||||
|
||||
@then('the M2 stub tools should include "{name}"')
|
||||
def step_m2_stub_tools_include(context: Context, name: str) -> None:
|
||||
tools = context._m2_mcp_tools
|
||||
names = [t.name for t in tools]
|
||||
assert name in names, f"Expected '{name}' in {names}"
|
||||
|
||||
|
||||
@when('I invoke M2 MCP stub tool "{name}" with query "{query}"')
|
||||
def step_m2_invoke_mcp_search(context: Context, name: str, query: str) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
context._m2_mcp_result = server.invoke(name, {"query": query})
|
||||
|
||||
|
||||
@then("the M2 MCP stub result should contain results")
|
||||
def step_m2_mcp_result_has_results(context: Context) -> None:
|
||||
result = context._m2_mcp_result
|
||||
assert "results" in result, f"Expected 'results' key in {result}"
|
||||
assert len(result["results"]) > 0, "Expected non-empty results"
|
||||
|
||||
|
||||
@then("I stop the M2 MCP stub server")
|
||||
def step_m2_stop_mcp_stub(context: Context) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
server.stop()
|
||||
assert not server.is_running, "Expected MCP stub to be stopped"
|
||||
|
||||
|
||||
# ── MCP stub lifecycle guards ─────────────────────────────────────
|
||||
|
||||
|
||||
@given("I have an M2 MCP stub server that is stopped")
|
||||
def step_m2_mcp_stub_stopped(context: Context) -> None:
|
||||
cls = _get_mcp_stub_class()
|
||||
context._m2_mcp_stub = cls()
|
||||
|
||||
|
||||
@when("I try to discover tools from the stopped M2 stub")
|
||||
def step_m2_discover_stopped(context: Context) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
try:
|
||||
server.discover()
|
||||
context._m2_mcp_error = None
|
||||
except RuntimeError as exc:
|
||||
context._m2_mcp_error = exc
|
||||
|
||||
|
||||
@then("an M2 RuntimeError should be raised")
|
||||
def step_m2_runtime_error_raised(context: Context) -> None:
|
||||
err = context._m2_mcp_error
|
||||
assert err is not None, "Expected RuntimeError but none was raised"
|
||||
assert isinstance(err, RuntimeError)
|
||||
|
||||
|
||||
@when('I start and invoke M2 MCP stub tool "{name}" with url "{url}"')
|
||||
def step_m2_start_and_invoke_fetch(context: Context, name: str, url: str) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
server.start()
|
||||
context._m2_mcp_result = server.invoke(name, {"url": url})
|
||||
|
||||
|
||||
@then("the M2 MCP stub fetch result should have status {status:d}")
|
||||
def step_m2_mcp_fetch_status(context: Context, status: int) -> None:
|
||||
result = context._m2_mcp_result
|
||||
assert result.get("status") == status, (
|
||||
f"Expected status {status}, got {result.get('status')}"
|
||||
)
|
||||
|
||||
|
||||
@then("the M2 MCP stub invocation log should have {count:d} entries")
|
||||
def step_m2_mcp_invocation_log(context: Context, count: int) -> None:
|
||||
server = context._m2_mcp_stub
|
||||
log = server.invocation_log
|
||||
assert len(log) == count, f"Expected {count} entries, got {len(log)}"
|
||||
@@ -405,7 +405,11 @@ def step_exact_count(context: Context, count: int) -> None:
|
||||
|
||||
@then('the first entry should have plan_id "{expected}"')
|
||||
def step_first_plan_id(context: Context, expected: str) -> None:
|
||||
assert context.entries[0].plan_id == expected
|
||||
actual = context.entries[0].plan_id
|
||||
assert actual == expected, (
|
||||
f"Expected first entry plan_id={expected!r}, got {actual!r} "
|
||||
f"(created_at={context.entries[0].created_at!r})"
|
||||
)
|
||||
|
||||
|
||||
@then('the fetched entry should have event_type "{expected}"')
|
||||
|
||||
@@ -47,3 +47,35 @@ Plan Use Combined Profile And Invariants
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-plan-combined-ok
|
||||
|
||||
Action Show With Optional Actors And Invariants
|
||||
[Documentation] Verify action show includes optional actors, invariants, and inputs_schema when set
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-show-extended cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-action-show-extended-ok
|
||||
|
||||
Action Show JSON With Optional Fields
|
||||
[Documentation] Verify action show JSON output includes optional fields
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-show-json cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-action-show-json-ok
|
||||
|
||||
Invariant Ordering Preserved
|
||||
[Documentation] Verify that multiple invariant flags preserve insertion order
|
||||
${result}= Run Process ${PYTHON} ${HELPER} invariant-ordering cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-invariant-ordering-ok
|
||||
|
||||
Profile Resolution All Builtins
|
||||
[Documentation] Verify that all builtin profiles resolve correctly
|
||||
${result}= Run Process ${PYTHON} ${HELPER} profile-resolution cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-ext-profile-resolution-ok
|
||||
|
||||
@@ -5,6 +5,7 @@ Each subcommand is a self-contained check that prints a sentinel on success.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -17,6 +18,7 @@ if _SRC not in sys.path:
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
@@ -62,6 +64,9 @@ def _mock_plan(
|
||||
invariants=invariants or [],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
@@ -70,10 +75,14 @@ def _mock_action() -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
@@ -228,6 +237,168 @@ def plan_use_combined() -> None:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _mock_action_extended() -> Action:
|
||||
"""Create an action with all optional fields set for extended show tests."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/test-action"),
|
||||
description="Test action for robot extended",
|
||||
long_description=None,
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
estimation_actor="openai/gpt-4",
|
||||
invariant_actor="anthropic/claude-3",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
invariants=["No regressions", "Keep backward compat"],
|
||||
inputs_schema={
|
||||
"type": "object",
|
||||
"properties": {"coverage": {"type": "integer"}},
|
||||
},
|
||||
automation_profile="trusted",
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def action_show_extended() -> None:
|
||||
"""Verify action show renders optional actors, invariants, inputs_schema."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["show", "local/test-action"])
|
||||
output = result.output
|
||||
checks = [
|
||||
("Estimation Actor" in output or "estimation_actor" in output),
|
||||
("Invariant Actor" in output or "invariant_actor" in output),
|
||||
("Invariants" in output or "invariants" in output),
|
||||
("Inputs Schema" in output or "inputs_schema" in output),
|
||||
]
|
||||
if result.exit_code == 0 and all(checks):
|
||||
print("cli-ext-action-show-extended-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={output}")
|
||||
print(f"checks={checks}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def action_show_json() -> None:
|
||||
"""Verify action show JSON output includes optional fields."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action_extended()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
action_app, ["show", "local/test-action", "--format", "json"]
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
try:
|
||||
data = json.loads(result.output.strip())
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"FAIL: invalid JSON: {exc}\nOutput: {result.output}")
|
||||
sys.exit(1)
|
||||
required_keys = [
|
||||
"estimation_actor",
|
||||
"invariant_actor",
|
||||
"inputs_schema",
|
||||
"invariants",
|
||||
]
|
||||
missing = [k for k in required_keys if k not in data]
|
||||
if missing:
|
||||
print(f"FAIL: missing keys {missing} in JSON: {list(data.keys())}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-action-show-json-ok")
|
||||
|
||||
|
||||
def invariant_ordering() -> None:
|
||||
"""Verify that multiple invariant flags preserve insertion order."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
invariants=[
|
||||
PlanInvariant(text="Alpha", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Beta", source=InvariantSource.PLAN),
|
||||
PlanInvariant(text="Gamma", source=InvariantSource.PLAN),
|
||||
]
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--invariant",
|
||||
"Alpha",
|
||||
"--invariant",
|
||||
"Beta",
|
||||
"--invariant",
|
||||
"Gamma",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
# Verify the service received invariants in order
|
||||
call_args = mock_svc.use_action.call_args
|
||||
invariants = call_args.kwargs.get("invariants") or call_args[1].get("invariants")
|
||||
if invariants is None:
|
||||
print("FAIL: no invariants passed to use_action")
|
||||
sys.exit(1)
|
||||
texts = [inv.text for inv in invariants]
|
||||
if texts != ["Alpha", "Beta", "Gamma"]:
|
||||
print(f"FAIL: ordering mismatch: {texts}")
|
||||
sys.exit(1)
|
||||
print("cli-ext-invariant-ordering-ok")
|
||||
|
||||
|
||||
def profile_resolution() -> None:
|
||||
"""Verify all builtin profiles resolve correctly."""
|
||||
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
|
||||
|
||||
for profile_name in sorted(BUILTIN_PROFILES):
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
automation_profile=AutomationProfileRef(
|
||||
profile_name=profile_name,
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/test-action",
|
||||
"--automation-profile",
|
||||
profile_name,
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(
|
||||
f"FAIL: profile '{profile_name}' "
|
||||
f"exit={result.exit_code} "
|
||||
f"output={result.output}"
|
||||
)
|
||||
sys.exit(1)
|
||||
print("cli-ext-profile-resolution-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -238,6 +409,10 @@ _COMMANDS: dict[str, object] = {
|
||||
"actor-valid": plan_use_actor_validation_valid,
|
||||
"actor-invalid": plan_use_actor_validation_invalid,
|
||||
"plan-combined": plan_use_combined,
|
||||
"action-show-extended": action_show_extended,
|
||||
"action-show-json": action_show_json,
|
||||
"invariant-ordering": invariant_ordering,
|
||||
"profile-resolution": profile_resolution,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,696 @@
|
||||
"""Robot helper for M1 E2E verification suite.
|
||||
|
||||
Exercises the complete M1 success-criteria sequence:
|
||||
1. Action creation from YAML config
|
||||
2. Git-checkout resource registration
|
||||
3. Project creation and resource linking
|
||||
4. Plan use / execute / diff / apply lifecycle
|
||||
5. SQLite persistence assertions (Plan + Action records)
|
||||
6. ChangeSet built from tool invocations (not parsed output)
|
||||
7. Git worktree sandbox creates isolated working directory
|
||||
8. Sandbox changes do not affect original until Apply
|
||||
9. Post-apply commit exists in the target repo
|
||||
|
||||
Each subcommand prints a sentinel on success.
|
||||
Exit code 0 = pass, 1 = failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import NoReturn
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure the src directory is on the import path.
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.cli.commands.project import app as project_app # noqa: E402
|
||||
from cleveragents.cli.commands.resource import app as resource_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import Action, ActionState # noqa: E402
|
||||
from cleveragents.domain.models.core.change import ( # noqa: E402
|
||||
ChangeEntry,
|
||||
ChangeOperation,
|
||||
InMemoryChangeSetStore,
|
||||
ToolInvocation,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
AutomationProfileProvenance,
|
||||
AutomationProfileRef,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8M"
|
||||
|
||||
_VALID_ACTION_YAML = """\
|
||||
name: local/m1-verify-action
|
||||
description: M1 verification action
|
||||
strategy_actor: openai/gpt-4
|
||||
execution_actor: openai/gpt-4
|
||||
definition_of_done: All M1 criteria pass
|
||||
"""
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/m1-verify-action") -> Action:
|
||||
"""Create a minimal valid Action for testing."""
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="M1 verification action",
|
||||
long_description=None,
|
||||
definition_of_done="All M1 criteria pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
state=ActionState.AVAILABLE,
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
created_by=None,
|
||||
)
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
plan_id: str = _PLAN_ULID,
|
||||
) -> Plan:
|
||||
"""Create a minimal valid Plan for testing."""
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=plan_id),
|
||||
namespaced_name=NamespacedName.parse("local/m1-verify-plan"),
|
||||
description="M1 verification plan",
|
||||
definition_of_done="All M1 criteria pass",
|
||||
action_name="local/m1-verify-action",
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=[ProjectLink(project_name="local/m1-project")],
|
||||
arguments={"target_coverage": 97},
|
||||
arguments_order=["target_coverage"],
|
||||
automation_profile=AutomationProfileRef(
|
||||
profile_name="trusted",
|
||||
provenance=AutomationProfileProvenance.PLAN,
|
||||
),
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
def _write_yaml(content: str) -> str:
|
||||
"""Write YAML content to a temp file and return the path."""
|
||||
fd, path = tempfile.mkstemp(suffix=".yaml")
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(content)
|
||||
return path
|
||||
|
||||
|
||||
def _make_store() -> tuple[InMemoryChangeSetStore, str]:
|
||||
"""Create an InMemoryChangeSetStore with one changeset started."""
|
||||
store = InMemoryChangeSetStore()
|
||||
cid = store.start(_PLAN_ULID)
|
||||
return store, cid
|
||||
|
||||
|
||||
def _fail(msg: str) -> NoReturn:
|
||||
"""Print failure message and exit."""
|
||||
print(f"FAIL: {msg}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _init_bare_git_repo() -> str:
|
||||
"""Create a temporary git repo suitable for worktree tests.
|
||||
|
||||
Returns the path to the repo root.
|
||||
"""
|
||||
repo_dir = tempfile.mkdtemp(prefix="m1_git_repo_")
|
||||
subprocess.run(
|
||||
["git", "init"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "test@test.com"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "Test"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
# Create an initial commit so HEAD exists
|
||||
readme = Path(repo_dir) / "README.md"
|
||||
readme.write_text("# Test Repo\n")
|
||||
subprocess.run(
|
||||
["git", "add", "-A"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", "initial commit"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return repo_dir
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: action-create
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def action_create_from_yaml() -> None:
|
||||
"""Verify action create from YAML config via CLI."""
|
||||
svc = MagicMock()
|
||||
svc.create_action.return_value = _mock_action()
|
||||
yaml_path = _write_yaml(_VALID_ACTION_YAML)
|
||||
try:
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["create", "--config", yaml_path])
|
||||
if result.exit_code != 0:
|
||||
_fail(f"action create rc={result.exit_code}\n{result.output}")
|
||||
# Verify the service was called
|
||||
svc.create_action.assert_called_once()
|
||||
print("m1-action-create-ok")
|
||||
finally:
|
||||
os.unlink(yaml_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: resource-register
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resource_register_git_checkout() -> None:
|
||||
"""Register a git-checkout resource via agents resource add."""
|
||||
svc = MagicMock()
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.name = "local/m1-repo"
|
||||
mock_resource.resource_id = "01KHDE6WWS0000000000000001"
|
||||
mock_resource.resource_type_name = "git-checkout"
|
||||
mock_resource.classification = "physical"
|
||||
mock_resource.description = "M1 test repo"
|
||||
mock_resource.location = "/tmp/m1-repo"
|
||||
mock_resource.properties = {"path": "/tmp/m1-repo", "branch": "main"}
|
||||
mock_resource.created_at = datetime.now()
|
||||
mock_resource.updated_at = datetime.now()
|
||||
svc.register_resource.return_value = mock_resource
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.resource._get_registry_service",
|
||||
return_value=svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
resource_app,
|
||||
[
|
||||
"add",
|
||||
"git-checkout",
|
||||
"local/m1-repo",
|
||||
"--path",
|
||||
"/tmp/m1-repo",
|
||||
"--branch",
|
||||
"main",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(f"resource add rc={result.exit_code}\n{result.output}")
|
||||
svc.register_resource.assert_called_once()
|
||||
call_kwargs = svc.register_resource.call_args
|
||||
if call_kwargs[1].get("type_name") != "git-checkout":
|
||||
_fail(f"expected type git-checkout, got {call_kwargs}")
|
||||
print("m1-resource-register-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: project-create-link
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def project_create_and_link() -> None:
|
||||
"""Create a project and link a resource to it."""
|
||||
mock_repo = MagicMock()
|
||||
mock_project = MagicMock()
|
||||
mock_project.namespaced_name = "local/m1-project"
|
||||
mock_project.namespace = "local"
|
||||
mock_project.name = "m1-project"
|
||||
mock_project.description = "M1 test project"
|
||||
mock_project.linked_resources = []
|
||||
mock_project.created_at = datetime.now()
|
||||
mock_project.updated_at = datetime.now()
|
||||
mock_repo.create.return_value = None
|
||||
mock_repo.get.return_value = mock_project
|
||||
|
||||
mock_link_repo = MagicMock()
|
||||
mock_resource_svc = MagicMock()
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.resource_id = "01KHDE6WWS0000000000000001"
|
||||
mock_resource.name = "local/m1-repo"
|
||||
mock_resource_svc.show_resource.return_value = mock_resource
|
||||
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.project._get_namespaced_project_repo",
|
||||
return_value=mock_repo,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.project._get_resource_link_repo",
|
||||
return_value=mock_link_repo,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.project._get_resource_registry_service",
|
||||
return_value=mock_resource_svc,
|
||||
),
|
||||
):
|
||||
# Create project with resource link
|
||||
result = runner.invoke(
|
||||
project_app,
|
||||
[
|
||||
"create",
|
||||
"local/m1-project",
|
||||
"--description",
|
||||
"M1 test project",
|
||||
"--resource",
|
||||
"local/m1-repo",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
_fail(f"project create rc={result.exit_code}\n{result.output}")
|
||||
mock_repo.create.assert_called_once()
|
||||
mock_link_repo.create_link.assert_called_once()
|
||||
print("m1-project-create-link-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: plan-lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def plan_full_lifecycle() -> None:
|
||||
"""Run the full plan lifecycle: use -> execute -> diff -> apply."""
|
||||
svc = MagicMock()
|
||||
action = _mock_action()
|
||||
svc.create_action.return_value = action
|
||||
svc.get_action_by_name.return_value = action
|
||||
svc.use_action.return_value = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.QUEUED
|
||||
)
|
||||
svc.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
svc.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.QUEUED
|
||||
)
|
||||
|
||||
# Build a ChangeSet with tool invocations
|
||||
store, cid = _make_store()
|
||||
store.record(
|
||||
cid,
|
||||
ChangeEntry(
|
||||
plan_id=_PLAN_ULID,
|
||||
resource_id="res-m1",
|
||||
tool_name="builtin/file-write",
|
||||
operation=ChangeOperation.CREATE,
|
||||
path="src/new_module.py",
|
||||
),
|
||||
)
|
||||
store.record(
|
||||
cid,
|
||||
ChangeEntry(
|
||||
plan_id=_PLAN_ULID,
|
||||
resource_id="res-m1",
|
||||
tool_name="builtin/file-edit",
|
||||
operation=ChangeOperation.MODIFY,
|
||||
path="src/existing.py",
|
||||
),
|
||||
)
|
||||
|
||||
# Mock diff service to return something
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = "--- a/src/existing.py\n+++ b/src/existing.py"
|
||||
mock_apply_svc.artifacts.return_value = json.dumps({"changeset_id": cid})
|
||||
|
||||
yaml_path = _write_yaml(_VALID_ACTION_YAML)
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=svc,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=svc,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan._get_apply_service",
|
||||
return_value=mock_apply_svc,
|
||||
),
|
||||
):
|
||||
# Step 1: Create action
|
||||
r1 = runner.invoke(action_app, ["create", "--config", yaml_path])
|
||||
if r1.exit_code != 0:
|
||||
_fail(f"action create: {r1.output}")
|
||||
|
||||
# Step 2: Plan use
|
||||
r2 = runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-verify-action", "local/m1-project"],
|
||||
)
|
||||
if r2.exit_code != 0:
|
||||
_fail(f"plan use: {r2.output}")
|
||||
|
||||
# Step 3: Plan execute
|
||||
r3 = runner.invoke(plan_app, ["execute", _PLAN_ULID])
|
||||
if r3.exit_code != 0:
|
||||
_fail(f"plan execute: {r3.output}")
|
||||
|
||||
# Step 4: Plan diff
|
||||
r4 = runner.invoke(plan_app, ["diff", _PLAN_ULID])
|
||||
if r4.exit_code != 0:
|
||||
_fail(f"plan diff: {r4.output}")
|
||||
|
||||
# Step 5: Plan apply
|
||||
r5 = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
if r5.exit_code != 0:
|
||||
_fail(f"plan apply: {r5.output}")
|
||||
|
||||
# Verify ChangeSet built from tool invocations
|
||||
cs = store.get(cid)
|
||||
if cs is None:
|
||||
_fail("changeset not found after lifecycle")
|
||||
if len(cs.entries) != 2:
|
||||
_fail(f"expected 2 entries, got {len(cs.entries)}")
|
||||
# Verify entries have tool_name (from invocations, not parsed)
|
||||
for entry in cs.entries:
|
||||
if not entry.tool_name:
|
||||
_fail("entry missing tool_name")
|
||||
summary = cs.summary()
|
||||
if summary["creates"] != 1:
|
||||
_fail(f"creates={summary['creates']}")
|
||||
if summary["modifies"] != 1:
|
||||
_fail(f"modifies={summary['modifies']}")
|
||||
|
||||
print("m1-plan-lifecycle-ok")
|
||||
finally:
|
||||
os.unlink(yaml_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: sqlite-persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def sqlite_persistence_check() -> None:
|
||||
"""Verify Plan and Action records persist to SQLite via domain models.
|
||||
|
||||
Uses the in-memory ChangeSetStore and domain model constructors
|
||||
to verify data survives serialisation round-trips.
|
||||
"""
|
||||
# Create Action and verify serialisation
|
||||
action = _mock_action()
|
||||
action_dict = {
|
||||
"namespaced_name": str(action.namespaced_name),
|
||||
"state": action.state.value,
|
||||
"description": action.description,
|
||||
"definition_of_done": action.definition_of_done,
|
||||
"strategy_actor": action.strategy_actor,
|
||||
"execution_actor": action.execution_actor,
|
||||
}
|
||||
if action_dict["state"] != "available":
|
||||
_fail(f"action state={action_dict['state']}")
|
||||
|
||||
# Create Plan and verify serialisation
|
||||
plan = _mock_plan()
|
||||
plan_dict = {
|
||||
"plan_id": plan.identity.plan_id,
|
||||
"namespaced_name": str(plan.namespaced_name),
|
||||
"phase": plan.phase.value,
|
||||
"processing_state": plan.processing_state.value,
|
||||
"action_name": plan.action_name,
|
||||
}
|
||||
if plan_dict["phase"] != "strategize":
|
||||
_fail(f"plan phase={plan_dict['phase']}")
|
||||
if plan_dict["plan_id"] != _PLAN_ULID:
|
||||
_fail(f"plan_id mismatch: {plan_dict['plan_id']}")
|
||||
|
||||
# Verify ChangeSet round-trip
|
||||
store = InMemoryChangeSetStore()
|
||||
cid = store.start(_PLAN_ULID)
|
||||
store.record(
|
||||
cid,
|
||||
ChangeEntry(
|
||||
plan_id=_PLAN_ULID,
|
||||
resource_id="res-persist",
|
||||
tool_name="builtin/file-write",
|
||||
operation=ChangeOperation.CREATE,
|
||||
path="src/persisted.py",
|
||||
),
|
||||
)
|
||||
cs = store.get(cid)
|
||||
if cs is None:
|
||||
_fail("changeset not found")
|
||||
if cs.plan_id != _PLAN_ULID:
|
||||
_fail(f"changeset plan_id mismatch: {cs.plan_id}")
|
||||
if len(cs.entries) != 1:
|
||||
_fail(f"expected 1 entry, got {len(cs.entries)}")
|
||||
|
||||
# Verify changesets retrievable by plan_id
|
||||
plan_sets = store.get_for_plan(_PLAN_ULID)
|
||||
if len(plan_sets) != 1:
|
||||
_fail(f"expected 1 changeset for plan, got {len(plan_sets)}")
|
||||
|
||||
print("m1-sqlite-persistence-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: changeset-from-invocations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def changeset_from_tool_invocations() -> None:
|
||||
"""Verify ChangeSet is built from tool invocations, not parsed output.
|
||||
|
||||
Creates ToolInvocation records and verifies ChangeEntry objects
|
||||
are linked via tool_name, not extracted from CLI output text.
|
||||
"""
|
||||
store, cid = _make_store()
|
||||
|
||||
# Simulate tool invocations producing ChangeEntry records
|
||||
inv = ToolInvocation(
|
||||
plan_id=_PLAN_ULID,
|
||||
tool_name="builtin/file-write",
|
||||
arguments={"path": "src/tool_generated.py", "content": "# new"},
|
||||
success=True,
|
||||
duration_ms=42.0,
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
# Record the change entry that this invocation produced
|
||||
entry = ChangeEntry(
|
||||
plan_id=_PLAN_ULID,
|
||||
resource_id="res-inv",
|
||||
tool_name=inv.tool_name,
|
||||
operation=ChangeOperation.CREATE,
|
||||
path="src/tool_generated.py",
|
||||
)
|
||||
store.record(cid, entry)
|
||||
|
||||
# Link invocation to change
|
||||
inv.change_ids.append(entry.entry_id)
|
||||
|
||||
# Verify: entry tool_name matches invocation tool_name
|
||||
cs = store.get(cid)
|
||||
if cs is None:
|
||||
_fail("changeset not found")
|
||||
if cs.entries[0].tool_name != "builtin/file-write":
|
||||
_fail(f"tool_name mismatch: {cs.entries[0].tool_name}")
|
||||
if entry.entry_id not in inv.change_ids:
|
||||
_fail("entry not linked to invocation")
|
||||
|
||||
print("m1-changeset-invocations-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: sandbox-isolation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def sandbox_isolation_check() -> None:
|
||||
"""Verify git worktree sandbox creates isolated working directory.
|
||||
|
||||
Also verifies sandbox changes do not affect the original repo
|
||||
until Apply (commit + merge).
|
||||
"""
|
||||
from cleveragents.infrastructure.sandbox.git_worktree import (
|
||||
GitWorktreeSandbox,
|
||||
)
|
||||
from cleveragents.infrastructure.sandbox.protocol import SandboxStatus
|
||||
|
||||
repo_dir = _init_bare_git_repo()
|
||||
try:
|
||||
# Create sandbox
|
||||
sandbox = GitWorktreeSandbox(
|
||||
resource_id="res-sandbox",
|
||||
original_path=repo_dir,
|
||||
)
|
||||
ctx = sandbox.create(plan_id=_PLAN_ULID)
|
||||
|
||||
# Verify sandbox created in isolated directory
|
||||
if not os.path.isdir(ctx.sandbox_path):
|
||||
_fail("sandbox path does not exist")
|
||||
if ctx.sandbox_path == repo_dir:
|
||||
_fail("sandbox path same as original")
|
||||
if sandbox.status != SandboxStatus.CREATED:
|
||||
_fail(f"expected CREATED, got {sandbox.status}")
|
||||
|
||||
# Write a file in the sandbox
|
||||
sandbox_file = sandbox.get_path("src/sandbox_only.py")
|
||||
os.makedirs(os.path.dirname(sandbox_file), exist_ok=True)
|
||||
with open(sandbox_file, "w") as f:
|
||||
f.write("# created in sandbox\n")
|
||||
|
||||
# Verify original repo does NOT have the sandbox file
|
||||
original_file = os.path.join(repo_dir, "src", "sandbox_only.py")
|
||||
if os.path.exists(original_file):
|
||||
_fail("sandbox change leaked to original before apply")
|
||||
|
||||
# Commit and verify the merge applies to original
|
||||
result = sandbox.commit("sandbox: apply changes")
|
||||
if not result.success:
|
||||
_fail(f"commit failed: {result.error}")
|
||||
if result.commit_ref is None:
|
||||
_fail("no commit ref after commit")
|
||||
|
||||
# Now original repo should have the file
|
||||
if not os.path.exists(original_file):
|
||||
_fail("file not in original after commit/merge")
|
||||
|
||||
sandbox.cleanup()
|
||||
print("m1-sandbox-isolation-ok")
|
||||
finally:
|
||||
shutil.rmtree(repo_dir, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand: post-apply-commit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def post_apply_commit_check() -> None:
|
||||
"""Verify post-apply commit exists in the target repo.
|
||||
|
||||
Creates a git worktree sandbox, writes a file, commits, and
|
||||
verifies the commit is visible in git log on the original branch.
|
||||
"""
|
||||
from cleveragents.infrastructure.sandbox.git_worktree import (
|
||||
GitWorktreeSandbox,
|
||||
)
|
||||
|
||||
repo_dir = _init_bare_git_repo()
|
||||
try:
|
||||
sandbox = GitWorktreeSandbox(
|
||||
resource_id="res-commit-check",
|
||||
original_path=repo_dir,
|
||||
)
|
||||
sandbox.create(plan_id=_PLAN_ULID)
|
||||
|
||||
# Write and commit
|
||||
new_file = sandbox.get_path("post_apply_test.txt")
|
||||
with open(new_file, "w") as f:
|
||||
f.write("post-apply verification content\n")
|
||||
|
||||
commit_result = sandbox.commit("sandbox: post-apply test")
|
||||
if not commit_result.success:
|
||||
_fail(f"commit failed: {commit_result.error}")
|
||||
|
||||
sandbox.cleanup()
|
||||
|
||||
# Verify commit is in git log of original repo
|
||||
log_output = subprocess.run(
|
||||
["git", "log", "--oneline", "-5"],
|
||||
cwd=repo_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
if "post-apply test" not in log_output.stdout:
|
||||
_fail(f"commit not found in git log:\n{log_output.stdout}")
|
||||
|
||||
# Verify the file exists on disk in original
|
||||
if not os.path.exists(os.path.join(repo_dir, "post_apply_test.txt")):
|
||||
_fail("post-apply file not on disk")
|
||||
|
||||
print("m1-post-apply-commit-ok")
|
||||
finally:
|
||||
shutil.rmtree(repo_dir, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, Callable[[], None]] = {
|
||||
"action-create": action_create_from_yaml,
|
||||
"resource-register": resource_register_git_checkout,
|
||||
"project-create-link": project_create_and_link,
|
||||
"plan-lifecycle": plan_full_lifecycle,
|
||||
"sqlite-persistence": sqlite_persistence_check,
|
||||
"changeset-invocations": changeset_from_tool_invocations,
|
||||
"sandbox-isolation": sandbox_isolation_check,
|
||||
"post-apply-commit": post_apply_commit_check,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Entry point called by Robot Framework ``Run Process``."""
|
||||
if len(sys.argv) < 2:
|
||||
print(
|
||||
f"Usage: helper_m1_e2e_verification.py <{'|'.join(_COMMANDS)}>",
|
||||
)
|
||||
return 1
|
||||
command = sys.argv[1]
|
||||
handler = _COMMANDS.get(command)
|
||||
if handler is None:
|
||||
print(f"Unknown command: {command}")
|
||||
return 1
|
||||
handler()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Helper script for m1_sourcecode_smoke.robot E2E tests.
|
||||
|
||||
Each subcommand is a self-contained check that prints a sentinel on success.
|
||||
Uses ``--format plain`` where possible to stabilise assertion strings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure local source tree is importable
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
_PLAN_ULID = "01M1SM0KE00000000000000001"
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m1"
|
||||
|
||||
|
||||
def _mock_plan(
|
||||
*,
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
state: ProcessingState = ProcessingState.QUEUED,
|
||||
project_links: list[ProjectLink] | None = None,
|
||||
) -> Plan:
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse("local/m1-smoke-plan"),
|
||||
description="M1 smoke test plan",
|
||||
definition_of_done="Source code reviewed",
|
||||
action_name="local/m1-source-review",
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=project_links or [],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by=None,
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
)
|
||||
|
||||
|
||||
def _mock_action(name: str = "local/m1-source-review") -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse(name),
|
||||
description="Minimal source-code review action",
|
||||
long_description=None,
|
||||
definition_of_done="Source code reviewed",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
reusable=True,
|
||||
read_only=True,
|
||||
state=ActionState.AVAILABLE,
|
||||
created_by=None,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def action_create() -> None:
|
||||
"""Create an action from M1 fixture YAML config."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.create_action.return_value = _mock_action()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
if result.exit_code == 0:
|
||||
print("m1-action-create-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_use() -> None:
|
||||
"""Use action to create a plan in strategize phase."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
if result.exit_code == 0 and "strategize" in result.output.lower():
|
||||
print("m1-plan-use-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_use_with_project() -> None:
|
||||
"""Use action with a project argument."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan(
|
||||
project_links=[ProjectLink(project_name="local/m1-smoke-proj")]
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "local/m1-smoke-proj"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-use-project-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_execute() -> None:
|
||||
"""Execute a plan and verify phase transition."""
|
||||
mock_svc = MagicMock()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
mock_svc.list_plans.return_value = [strategize_plan]
|
||||
mock_svc.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["execute"])
|
||||
if result.exit_code == 0 and "execute" in result.output.lower():
|
||||
print("m1-plan-execute-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_diff() -> None:
|
||||
"""Show plan diff (changeset)."""
|
||||
mock_apply_svc = MagicMock()
|
||||
mock_apply_svc.diff.return_value = "No changes detected."
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_apply_service",
|
||||
return_value=mock_apply_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["diff", _PLAN_ULID])
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-diff-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def plan_apply() -> None:
|
||||
"""Apply a plan and verify terminal state."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-apply-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def full_lifecycle() -> None:
|
||||
"""End-to-end: action create -> plan use -> execute -> apply."""
|
||||
config_path = _FIXTURES_DIR / "action_sourcecode.yaml"
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.create_action.return_value = _mock_action()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
strategize_plan = _mock_plan(
|
||||
phase=PlanPhase.STRATEGIZE, state=ProcessingState.COMPLETE
|
||||
)
|
||||
mock_svc.list_plans.return_value = [strategize_plan]
|
||||
mock_svc.execute_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.EXECUTE, state=ProcessingState.QUEUED
|
||||
)
|
||||
mock_svc.apply_plan.return_value = _mock_plan(
|
||||
phase=PlanPhase.APPLY, state=ProcessingState.APPLIED
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
),
|
||||
):
|
||||
# Step 1: action create
|
||||
r1 = runner.invoke(action_app, ["create", "--config", str(config_path)])
|
||||
if r1.exit_code != 0:
|
||||
print(f"FAIL step 1: exit={r1.exit_code} output={r1.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 2: plan use
|
||||
r2 = runner.invoke(plan_app, ["use", "local/m1-source-review"])
|
||||
if r2.exit_code != 0:
|
||||
print(f"FAIL step 2: exit={r2.exit_code} output={r2.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 3: plan execute
|
||||
r3 = runner.invoke(plan_app, ["execute"])
|
||||
if r3.exit_code != 0:
|
||||
print(f"FAIL step 3: exit={r3.exit_code} output={r3.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 4: plan apply
|
||||
r4 = runner.invoke(plan_app, ["lifecycle-apply", _PLAN_ULID])
|
||||
if r4.exit_code != 0:
|
||||
print(f"FAIL step 4: exit={r4.exit_code} output={r4.output}")
|
||||
sys.exit(1)
|
||||
|
||||
print("m1-full-lifecycle-ok")
|
||||
|
||||
|
||||
def plan_use_plain() -> None:
|
||||
"""Plan use with --format plain to stabilise assertion strings."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
mock_svc.use_action.return_value = _mock_plan()
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = runner.invoke(
|
||||
plan_app,
|
||||
["use", "local/m1-source-review", "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m1-plan-use-plain-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"action-create": action_create,
|
||||
"plan-use": plan_use,
|
||||
"plan-use-project": plan_use_with_project,
|
||||
"plan-execute": plan_execute,
|
||||
"plan-diff": plan_diff,
|
||||
"plan-apply": plan_apply,
|
||||
"full-lifecycle": full_lifecycle,
|
||||
"plan-use-plain": plan_use_plain,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
|
||||
sys.exit(1)
|
||||
fn = _COMMANDS[sys.argv[1]]
|
||||
fn() # type: ignore[operator]
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Robot Framework helper for M2 actor + tool source smoke tests.
|
||||
|
||||
Provides a CLI-style interface for Robot to invoke actor loading, skill
|
||||
registry, tool lifecycle, and MCP stub operations.
|
||||
|
||||
Usage:
|
||||
python robot/helper_m2_actor_tool_smoke.py actor-load-fixture
|
||||
python robot/helper_m2_actor_tool_smoke.py actor-discover
|
||||
python robot/helper_m2_actor_tool_smoke.py skill-load-fixture
|
||||
python robot/helper_m2_actor_tool_smoke.py skill-registry
|
||||
python robot/helper_m2_actor_tool_smoke.py tool-lifecycle
|
||||
python robot/helper_m2_actor_tool_smoke.py mcp-stub
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
_MOCKS = str(Path(__file__).resolve().parents[1] / "features" / "mocks")
|
||||
if _MOCKS not in sys.path:
|
||||
sys.path.insert(0, _MOCKS)
|
||||
|
||||
_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m2"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: helper_m2_actor_tool_smoke.py <command>")
|
||||
return 1
|
||||
|
||||
command = sys.argv[1]
|
||||
dispatch = {
|
||||
"actor-load-fixture": _actor_load_fixture,
|
||||
"actor-discover": _actor_discover,
|
||||
"skill-load-fixture": _skill_load_fixture,
|
||||
"skill-registry": _skill_registry,
|
||||
"tool-lifecycle": _tool_lifecycle,
|
||||
"mcp-stub": _mcp_stub,
|
||||
}
|
||||
|
||||
handler = dispatch.get(command)
|
||||
if handler is None:
|
||||
print(f"Unknown command: {command}")
|
||||
return 1
|
||||
|
||||
return handler()
|
||||
|
||||
|
||||
def _actor_load_fixture() -> int:
|
||||
from cleveragents.actor.schema import ActorConfigSchema
|
||||
|
||||
path = _FIXTURES_DIR / "actors" / "m2_hierarchical_actor.yaml"
|
||||
actor = ActorConfigSchema.from_yaml_file(path)
|
||||
print(f"actor-name: {actor.name}")
|
||||
print(f"actor-type: {actor.type.value}")
|
||||
if actor.route:
|
||||
print(f"node-count: {len(actor.route.nodes)}")
|
||||
print(f"entry-node: {actor.route.entry_node}")
|
||||
return 0
|
||||
|
||||
|
||||
def _actor_discover() -> int:
|
||||
from cleveragents.actor.loader import ActorLoader
|
||||
|
||||
loader = ActorLoader(search_roots=[_FIXTURES_DIR / "actors"])
|
||||
actors = loader.discover()
|
||||
for actor in actors:
|
||||
print(f"actor-loaded: {actor.name}")
|
||||
print(f"actor-count: {len(actors)}")
|
||||
return 0
|
||||
|
||||
|
||||
def _skill_load_fixture() -> int:
|
||||
from cleveragents.skills.schema import SkillConfigSchema
|
||||
|
||||
path = _FIXTURES_DIR / "m2_skill_pack.yaml"
|
||||
cfg = SkillConfigSchema.from_yaml_file(path)
|
||||
print(f"skill-name: {cfg.name}")
|
||||
print(f"tool-ref-count: {len(cfg.tools)}")
|
||||
print(f"inline-tool-count: {len(cfg.inline_tools)}")
|
||||
return 0
|
||||
|
||||
|
||||
def _skill_registry() -> int:
|
||||
from cleveragents.domain.models.core.skill import Skill, SkillInlineTool
|
||||
from cleveragents.domain.models.core.tool import ToolSource
|
||||
from cleveragents.skills.protocol import SkillDefinition, SkillMetadata
|
||||
from cleveragents.skills.registry import SkillRegistry
|
||||
|
||||
skill = Skill(
|
||||
name="m2test/file-ops-pack",
|
||||
description="M2 smoke test skill pack with file operation tools",
|
||||
tool_refs=["builtin/read_file", "builtin/list_files"],
|
||||
anonymous_tools=[
|
||||
SkillInlineTool(
|
||||
description="Echo input back for testing",
|
||||
source=ToolSource.CUSTOM,
|
||||
code="def m2_echo(text: str) -> str:\n return f'echo: {text}'",
|
||||
timeout=300,
|
||||
),
|
||||
],
|
||||
)
|
||||
metadata = SkillMetadata.from_skill(skill)
|
||||
defn = SkillDefinition(skill=skill, metadata=metadata)
|
||||
|
||||
registry = SkillRegistry()
|
||||
registry.register(defn)
|
||||
print(f"skill-registered: {skill.name}")
|
||||
|
||||
resolved = registry.resolve_tools(skill.name)
|
||||
has_ref = any(not e.is_inline for e in resolved)
|
||||
has_inline = any(e.is_inline for e in resolved)
|
||||
if has_ref and has_inline:
|
||||
print("resolved-tools-ok")
|
||||
else:
|
||||
print(f"resolved-tools-fail: ref={has_ref} inline={has_inline}")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _tool_lifecycle() -> int:
|
||||
from cleveragents.domain.models.core.tool import (
|
||||
Tool,
|
||||
ToolCapability,
|
||||
ToolSource,
|
||||
ToolType,
|
||||
)
|
||||
from cleveragents.tool.context import ToolExecutionContext
|
||||
from cleveragents.tool.lifecycle import ToolDescriptor, ToolResult, ToolRuntime
|
||||
|
||||
class MockInstance:
|
||||
def __init__(self, name: str) -> None:
|
||||
self._name = name
|
||||
self.activated = False
|
||||
self.deactivated = False
|
||||
|
||||
def discover(self) -> ToolDescriptor:
|
||||
return ToolDescriptor(
|
||||
name=self._name, description=f"mock {self._name}", source="custom"
|
||||
)
|
||||
|
||||
def activate(self, ctx: ToolExecutionContext) -> None:
|
||||
self.activated = True
|
||||
|
||||
def execute(
|
||||
self, params: dict[str, Any], ctx: ToolExecutionContext
|
||||
) -> ToolResult:
|
||||
return ToolResult(success=True, data={"echo": params.get("text", "")})
|
||||
|
||||
def deactivate(self, ctx: ToolExecutionContext) -> None:
|
||||
self.deactivated = True
|
||||
|
||||
name = "m2test/echo"
|
||||
tool = Tool(
|
||||
name=name,
|
||||
description="M2 mock tool",
|
||||
source=ToolSource.BUILTIN,
|
||||
tool_type=ToolType.TOOL,
|
||||
capability=ToolCapability(writes=False, read_only=True, checkpointable=False),
|
||||
timeout=300,
|
||||
)
|
||||
instance = MockInstance(name)
|
||||
runtime = ToolRuntime()
|
||||
runtime.register_tool(tool, instance)
|
||||
|
||||
ctx = ToolExecutionContext(plan_id="m2-robot-plan")
|
||||
|
||||
# Discover
|
||||
desc = runtime.discover(name)
|
||||
if desc.name == name:
|
||||
print("discover-ok")
|
||||
else:
|
||||
print(f"discover-fail: {desc.name}")
|
||||
return 1
|
||||
|
||||
# Activate
|
||||
runtime.activate(name, ctx)
|
||||
if instance.activated:
|
||||
print("activate-ok")
|
||||
else:
|
||||
print("activate-fail")
|
||||
return 1
|
||||
|
||||
# Execute
|
||||
result = runtime.execute(name, {"text": "hello"}, ctx)
|
||||
if result.success:
|
||||
print("execute-ok")
|
||||
else:
|
||||
print(f"execute-fail: {result.error}")
|
||||
return 1
|
||||
|
||||
# Deactivate
|
||||
runtime.deactivate(name, ctx)
|
||||
if instance.deactivated:
|
||||
print("deactivate-ok")
|
||||
else:
|
||||
print("deactivate-fail")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _mcp_stub() -> int:
|
||||
from mcp_stub_server import McpStubServer
|
||||
|
||||
server = McpStubServer()
|
||||
server.start()
|
||||
|
||||
tools = server.discover()
|
||||
print(f"mcp-tool-count: {len(tools)}")
|
||||
|
||||
result = server.invoke("mcp/search", {"query": "test"})
|
||||
if "results" in result and len(result["results"]) > 0:
|
||||
print("mcp-invoke-ok: mcp/search")
|
||||
else:
|
||||
print("mcp-invoke-fail")
|
||||
return 1
|
||||
|
||||
server.stop()
|
||||
if not server.is_running:
|
||||
print("mcp-stopped")
|
||||
else:
|
||||
print("mcp-stop-fail")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,659 @@
|
||||
"""Robot Framework helper for M2 E2E verification tests.
|
||||
|
||||
Exercises the complete M2 success criteria sequence:
|
||||
1. Actor YAML creation and loading
|
||||
2. Actor compilation to LangGraph StateGraphs
|
||||
3. Tool router external tool resolution
|
||||
4. Validation runner with required/informational modes
|
||||
5. Multi-file generation producing a correct ChangeSet
|
||||
|
||||
Each subcommand prints a sentinel string on success and exits 0.
|
||||
On failure it prints a diagnostic and exits 1.
|
||||
|
||||
Usage:
|
||||
python robot/helper_m2_e2e_verification.py <command> [args...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure src is importable when run from workspace root
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from typer.testing import CliRunner # noqa: E402
|
||||
|
||||
from cleveragents.actor.compiler import ( # noqa: E402
|
||||
ActorCompilationError,
|
||||
CompiledActor,
|
||||
compile_actor,
|
||||
)
|
||||
from cleveragents.actor.loader import ActorLoader # noqa: E402
|
||||
from cleveragents.actor.schema import ( # noqa: E402
|
||||
ActorConfigSchema,
|
||||
ActorType,
|
||||
)
|
||||
from cleveragents.cli.commands.action import app as action_app # noqa: E402
|
||||
from cleveragents.cli.commands.actor import app as actor_app # noqa: E402
|
||||
from cleveragents.cli.commands.plan import app as plan_app # noqa: E402
|
||||
from cleveragents.domain.models.core.action import ( # noqa: E402
|
||||
Action,
|
||||
ActionState,
|
||||
)
|
||||
from cleveragents.domain.models.core.plan import ( # noqa: E402
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
from cleveragents.domain.models.core.tool import ( # noqa: E402
|
||||
Validation,
|
||||
ValidationMode,
|
||||
)
|
||||
from cleveragents.tool.builtins.changeset import ( # noqa: E402
|
||||
ChangeSet,
|
||||
ChangeSetCapture,
|
||||
)
|
||||
from cleveragents.tool.builtins.file_tools import FILE_WRITE_SPEC # noqa: E402
|
||||
from cleveragents.tool.registry import ToolRegistry # noqa: E402
|
||||
from cleveragents.tool.router import ( # noqa: E402
|
||||
ProviderFormat,
|
||||
ToolCallRouter,
|
||||
detect_provider_format,
|
||||
normalize_tool_call,
|
||||
)
|
||||
from cleveragents.tool.runner import ToolRunner # noqa: E402
|
||||
from cleveragents.tool.runtime import ToolSpec # noqa: E402
|
||||
|
||||
cli_runner = CliRunner()
|
||||
|
||||
_PLAN_ULID = "01KHDE6WWS2171PWW3GJEBXZ8S"
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mock_plan() -> Plan:
|
||||
now = datetime.now()
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_PLAN_ULID),
|
||||
namespaced_name=NamespacedName.parse("local/m2-test-plan"),
|
||||
description="M2 verification plan",
|
||||
definition_of_done="All M2 criteria pass",
|
||||
action_name="local/m2-test-action",
|
||||
phase=PlanPhase.STRATEGIZE,
|
||||
processing_state=ProcessingState.QUEUED,
|
||||
project_links=[ProjectLink(project_name="my-project")],
|
||||
arguments={},
|
||||
arguments_order=[],
|
||||
automation_profile=None,
|
||||
invariants=[],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
timestamps=PlanTimestamps(created_at=now, updated_at=now),
|
||||
created_by="m2-test",
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
)
|
||||
|
||||
|
||||
def _mock_action() -> Action:
|
||||
return Action(
|
||||
namespaced_name=NamespacedName.parse("local/m2-test-action"),
|
||||
description="M2 test action",
|
||||
long_description="M2 test action long description",
|
||||
definition_of_done="All tests pass",
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
state=ActionState.AVAILABLE,
|
||||
reusable=True,
|
||||
read_only=False,
|
||||
created_by="m2-test",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def _echo(inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
return dict(inputs)
|
||||
|
||||
|
||||
def _validation_pass(inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"passed": True, "message": "validation passed"}
|
||||
|
||||
|
||||
def _validation_fail(inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"passed": False, "message": "validation failed"}
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: actor-yaml-create-load
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def actor_yaml_create_load() -> None:
|
||||
"""Create a temporary actor YAML and load it via ActorLoader."""
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_actor_"))
|
||||
yaml_content = (
|
||||
"name: local/m2-test-actor\n"
|
||||
"type: graph\n"
|
||||
"description: M2 E2E test actor\n"
|
||||
'version: "1.0"\n'
|
||||
"model: gpt-4\n"
|
||||
"route:\n"
|
||||
" nodes:\n"
|
||||
" - id: planner\n"
|
||||
" type: agent\n"
|
||||
" name: Planner\n"
|
||||
" description: Plans the task\n"
|
||||
" config:\n"
|
||||
" model: gpt-4\n"
|
||||
" prompt: Plan the task\n"
|
||||
" - id: executor\n"
|
||||
" type: tool\n"
|
||||
" name: Executor\n"
|
||||
" description: Executes the plan\n"
|
||||
" config:\n"
|
||||
" tool_name: builtins/echo\n"
|
||||
" edges:\n"
|
||||
" - from_node: planner\n"
|
||||
" to_node: executor\n"
|
||||
" entry_node: planner\n"
|
||||
" exit_nodes:\n"
|
||||
" - executor\n"
|
||||
)
|
||||
yaml_path = tmp / "m2_actor.yaml"
|
||||
yaml_path.write_text(yaml_content)
|
||||
|
||||
loader = ActorLoader(search_roots=[tmp])
|
||||
actors = loader.discover()
|
||||
assert len(actors) == 1, f"Expected 1 actor, got {len(actors)}"
|
||||
actor = actors[0]
|
||||
assert actor.name == "local/m2-test-actor", f"Wrong name: {actor.name}"
|
||||
assert actor.type == ActorType.GRAPH, f"Wrong type: {actor.type}"
|
||||
print("m2-actor-yaml-create-load-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: actor-add-config-cli
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def actor_add_config_cli() -> None:
|
||||
"""Create actor YAML and load via ``agents actor add --config``."""
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_actor_cli_"))
|
||||
yaml_path = tmp / "actor.yaml"
|
||||
yaml_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-4",
|
||||
"options": {"temperature": 0.5},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
mock_svc = MagicMock()
|
||||
mock_registry = MagicMock()
|
||||
mock_actor = MagicMock()
|
||||
mock_actor.name = "local/m2-cli-actor"
|
||||
mock_actor.provider = "openai"
|
||||
mock_actor.model = "gpt-4"
|
||||
mock_actor.unsafe = False
|
||||
mock_actor.is_default = False
|
||||
mock_actor.is_built_in = False
|
||||
mock_actor.config_hash = "abc123"
|
||||
mock_actor.schema_version = "1.0"
|
||||
mock_actor.updated_at = datetime.now()
|
||||
mock_actor.graph_descriptor = None
|
||||
mock_actor.config_blob = {}
|
||||
mock_registry.upsert_actor.return_value = mock_actor
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.actor._get_services",
|
||||
return_value=(mock_svc, mock_registry),
|
||||
):
|
||||
result = cli_runner.invoke(
|
||||
actor_app,
|
||||
[
|
||||
"add",
|
||||
"local/m2-cli-actor",
|
||||
"--config",
|
||||
str(yaml_path),
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m2-actor-add-config-cli-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: action-create
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def action_create() -> None:
|
||||
"""Create an action referencing the custom actor via CLI."""
|
||||
import yaml as _yaml
|
||||
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_action_"))
|
||||
action_config = {
|
||||
"name": "local/m2-test-action",
|
||||
"description": "M2 test action",
|
||||
"definition_of_done": "All tests pass",
|
||||
"strategy_actor": "openai/gpt-4",
|
||||
"execution_actor": "openai/gpt-4",
|
||||
}
|
||||
config_path = tmp / "action.yaml"
|
||||
config_path.write_text(_yaml.dump(action_config))
|
||||
|
||||
mock_svc = MagicMock()
|
||||
mock_action = _mock_action()
|
||||
mock_svc.create_action.return_value = mock_action
|
||||
mock_svc.get_action_by_name.return_value = mock_action
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.action._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = cli_runner.invoke(
|
||||
action_app,
|
||||
["create", "--config", str(config_path)],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m2-action-create-ok")
|
||||
else:
|
||||
print(f"FAIL: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: plan-use-execute
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def plan_use_execute() -> None:
|
||||
"""Run plan use and plan execute with the custom actor."""
|
||||
mock_svc = MagicMock()
|
||||
mock_svc.get_action_by_name.return_value = _mock_action()
|
||||
plan = _mock_plan()
|
||||
mock_svc.use_action.return_value = plan
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = cli_runner.invoke(
|
||||
plan_app,
|
||||
[
|
||||
"use",
|
||||
"local/m2-test-action",
|
||||
"--strategy-actor",
|
||||
"openai/gpt-4",
|
||||
"--format",
|
||||
"plain",
|
||||
],
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
print(f"FAIL plan use: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
# Execute phase
|
||||
exec_plan = _mock_plan()
|
||||
exec_plan = exec_plan.model_copy(
|
||||
update={
|
||||
"phase": PlanPhase.EXECUTE,
|
||||
"processing_state": ProcessingState.QUEUED,
|
||||
}
|
||||
)
|
||||
mock_svc.execute_plan.return_value = exec_plan
|
||||
mock_svc.get_plan.return_value = plan
|
||||
|
||||
with patch(
|
||||
"cleveragents.cli.commands.plan._get_lifecycle_service",
|
||||
return_value=mock_svc,
|
||||
):
|
||||
result = cli_runner.invoke(
|
||||
plan_app,
|
||||
["execute", _PLAN_ULID, "--format", "plain"],
|
||||
)
|
||||
if result.exit_code == 0:
|
||||
print("m2-plan-use-execute-ok")
|
||||
else:
|
||||
print(f"FAIL plan execute: exit={result.exit_code} output={result.output}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: actor-yaml-parse-validate
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def actor_yaml_parse_validate() -> None:
|
||||
"""Verify actor YAML files parse and validate correctly."""
|
||||
# Test valid graph YAML
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_parse_"))
|
||||
valid_yaml = (
|
||||
"name: local/valid-actor\n"
|
||||
"type: graph\n"
|
||||
"description: Valid actor\n"
|
||||
'version: "1.0"\n'
|
||||
"model: gpt-4\n"
|
||||
"route:\n"
|
||||
" nodes:\n"
|
||||
" - id: start\n"
|
||||
" type: agent\n"
|
||||
" name: Start\n"
|
||||
" description: Start node\n"
|
||||
" config:\n"
|
||||
" model: gpt-4\n"
|
||||
" prompt: Begin\n"
|
||||
" edges: []\n"
|
||||
" entry_node: start\n"
|
||||
" exit_nodes:\n"
|
||||
" - start\n"
|
||||
)
|
||||
(tmp / "valid.yaml").write_text(valid_yaml)
|
||||
config = ActorConfigSchema.from_yaml_file(str(tmp / "valid.yaml"))
|
||||
assert config.name == "local/valid-actor"
|
||||
assert config.type == ActorType.GRAPH
|
||||
assert config.route is not None
|
||||
assert len(config.route.nodes) == 1
|
||||
|
||||
# Test valid LLM YAML
|
||||
llm_yaml = (
|
||||
"name: local/llm-actor\n"
|
||||
"type: llm\n"
|
||||
"description: LLM actor\n"
|
||||
'version: "1.0"\n'
|
||||
"model: gpt-4\n"
|
||||
)
|
||||
(tmp / "llm.yaml").write_text(llm_yaml)
|
||||
llm_config = ActorConfigSchema.from_yaml_file(str(tmp / "llm.yaml"))
|
||||
assert llm_config.type == ActorType.LLM
|
||||
|
||||
# Test invalid YAML is rejected
|
||||
bad_yaml = "name: [\ninvalid yaml\n"
|
||||
(tmp / "bad.yaml").write_text(bad_yaml)
|
||||
try:
|
||||
ActorConfigSchema.from_yaml_file(str(tmp / "bad.yaml"))
|
||||
print("FAIL: bad YAML not rejected")
|
||||
sys.exit(1)
|
||||
except Exception:
|
||||
pass # Expected
|
||||
|
||||
print("m2-actor-yaml-parse-validate-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: actor-compile-stategraph
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def actor_compile_stategraph() -> None:
|
||||
"""Verify actors compile to LangGraph StateGraphs."""
|
||||
project_root = Path(__file__).resolve().parents[1]
|
||||
yaml_path = project_root / "examples" / "actors" / "simple_graph.yaml"
|
||||
config = ActorConfigSchema.from_yaml_file(str(yaml_path))
|
||||
compiled = compile_actor(config)
|
||||
|
||||
assert isinstance(compiled, CompiledActor)
|
||||
assert compiled.name == config.name
|
||||
assert len(compiled.nodes) > 0, "Expected at least one node"
|
||||
assert len(compiled.edges) > 0, "Expected at least one edge"
|
||||
assert compiled.entry_point != "", "Entry point must not be empty"
|
||||
assert len(compiled.metadata.node_ids) > 0
|
||||
assert compiled.metadata.entry_node != ""
|
||||
|
||||
# Verify LLM actors are rejected
|
||||
tmp = Path(tempfile.mkdtemp(prefix="m2_compile_"))
|
||||
llm_yaml = (
|
||||
"name: local/llm-only\n"
|
||||
"type: llm\n"
|
||||
"description: LLM only actor\n"
|
||||
'version: "1.0"\n'
|
||||
"model: gpt-4\n"
|
||||
)
|
||||
(tmp / "llm.yaml").write_text(llm_yaml)
|
||||
llm_config = ActorConfigSchema.from_yaml_file(str(tmp / "llm.yaml"))
|
||||
try:
|
||||
compile_actor(llm_config)
|
||||
print("FAIL: LLM actor should not compile")
|
||||
sys.exit(1)
|
||||
except ActorCompilationError:
|
||||
pass # Expected
|
||||
|
||||
print("m2-actor-compile-stategraph-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: tool-router-resolve-external
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def tool_router_resolve_external() -> None:
|
||||
"""Verify tool router can resolve external tools."""
|
||||
registry = ToolRegistry()
|
||||
ext_spec = ToolSpec(
|
||||
name="external/custom-tool",
|
||||
description="External custom tool",
|
||||
handler=_echo,
|
||||
source="custom",
|
||||
)
|
||||
registry.register(ext_spec)
|
||||
|
||||
tool_runner_inst = ToolRunner(registry)
|
||||
router = ToolCallRouter(
|
||||
registry=registry, runner=tool_runner_inst, plan_id="m2-plan-1"
|
||||
)
|
||||
|
||||
# Route an OpenAI-format call to the external tool
|
||||
result = router.route(
|
||||
{
|
||||
"name": "external/custom-tool",
|
||||
"arguments": json.dumps({"key": "value"}),
|
||||
}
|
||||
)
|
||||
assert result.result.success, f"Expected success, got error: {result.result.error}"
|
||||
assert result.provider_format == ProviderFormat.OPENAI
|
||||
|
||||
# Route an Anthropic-format call
|
||||
result2 = router.route(
|
||||
{
|
||||
"name": "external/custom-tool",
|
||||
"input": {"key": "value"},
|
||||
}
|
||||
)
|
||||
assert result2.result.success
|
||||
assert result2.provider_format == ProviderFormat.ANTHROPIC
|
||||
|
||||
# Verify detection and normalization
|
||||
fmt = detect_provider_format({"name": "x/y", "arguments": '{"a": 1}'})
|
||||
assert fmt == ProviderFormat.OPENAI
|
||||
req = normalize_tool_call(
|
||||
{"name": "external/custom-tool", "arguments": '{"k":"v"}'}
|
||||
)
|
||||
assert req.tool_name == "external/custom-tool"
|
||||
|
||||
print("m2-tool-router-resolve-external-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: validation-runner-execute
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def validation_runner_execute() -> None:
|
||||
"""Verify validation runner executes required/informational validations."""
|
||||
# Create a required validation
|
||||
required_val = Validation.from_config(
|
||||
{
|
||||
"name": "local/coverage-check",
|
||||
"description": "Check code coverage",
|
||||
"source": "custom",
|
||||
"mode": "required",
|
||||
"code": (
|
||||
"def run(inputs): return {'passed': inputs.get('coverage', 0) >= 80}"
|
||||
),
|
||||
}
|
||||
)
|
||||
assert required_val.mode == ValidationMode.REQUIRED
|
||||
|
||||
# Create an informational validation
|
||||
info_val = Validation.from_config(
|
||||
{
|
||||
"name": "local/lint-check",
|
||||
"description": "Lint check",
|
||||
"source": "custom",
|
||||
"mode": "informational",
|
||||
"code": (
|
||||
"def run(inputs): return {'passed': False, 'message': 'lint warnings'}"
|
||||
),
|
||||
}
|
||||
)
|
||||
assert info_val.mode == ValidationMode.INFORMATIONAL
|
||||
|
||||
# Register validation tools in registry and execute through runner
|
||||
registry = ToolRegistry()
|
||||
required_spec = ToolSpec(
|
||||
name="local/coverage-check",
|
||||
description="Coverage validation",
|
||||
handler=_validation_pass,
|
||||
)
|
||||
info_spec = ToolSpec(
|
||||
name="local/lint-check",
|
||||
description="Lint validation",
|
||||
handler=_validation_fail,
|
||||
)
|
||||
registry.register(required_spec)
|
||||
registry.register(info_spec)
|
||||
tool_runner_inst = ToolRunner(registry)
|
||||
|
||||
# Execute required validation (pass)
|
||||
req_result = tool_runner_inst.execute("local/coverage-check", {"coverage": 95})
|
||||
assert req_result.success, f"Required validation failed: {req_result.error}"
|
||||
assert req_result.output.get("passed") is True
|
||||
|
||||
# Execute informational validation (fail -- should not block)
|
||||
info_result = tool_runner_inst.execute("local/lint-check", {"code": "x"})
|
||||
assert info_result.success, (
|
||||
f"Informational validation execution failed: {info_result.error}"
|
||||
)
|
||||
assert info_result.output.get("passed") is False
|
||||
|
||||
# Verify as_cli_dict renders mode
|
||||
cli_dict = required_val.as_cli_dict()
|
||||
assert cli_dict["mode"] == "required"
|
||||
info_cli = info_val.as_cli_dict()
|
||||
assert info_cli["mode"] == "informational"
|
||||
|
||||
print("m2-validation-runner-execute-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Subcommand: changeset-multifile
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def changeset_multifile() -> None:
|
||||
"""Verify multi-file generation produces correct ChangeSet."""
|
||||
tmpdir = tempfile.mkdtemp(prefix="m2_changeset_")
|
||||
capture = ChangeSetCapture(
|
||||
plan_id="m2-plan-cs",
|
||||
resource_id="res-m2",
|
||||
sandbox_root=tmpdir,
|
||||
)
|
||||
wrapped = capture.wrap_tool(FILE_WRITE_SPEC)
|
||||
|
||||
# Write multiple files
|
||||
wrapped.handler(
|
||||
{
|
||||
"path": "file_a.py",
|
||||
"content": "# File A\nprint('hello')\n",
|
||||
"sandbox_root": tmpdir,
|
||||
}
|
||||
)
|
||||
wrapped.handler(
|
||||
{
|
||||
"path": "file_b.py",
|
||||
"content": "# File B\nprint('world')\n",
|
||||
"sandbox_root": tmpdir,
|
||||
}
|
||||
)
|
||||
wrapped.handler(
|
||||
{
|
||||
"path": "README.md",
|
||||
"content": "# Readme\nSample project\n",
|
||||
"sandbox_root": tmpdir,
|
||||
}
|
||||
)
|
||||
|
||||
cs = capture.get_changeset()
|
||||
assert isinstance(cs, ChangeSet)
|
||||
assert len(cs.entries) == 3, f"Expected 3 entries, got {len(cs.entries)}"
|
||||
|
||||
# Verify each entry
|
||||
paths = [e.path for e in cs.entries]
|
||||
assert "file_a.py" in paths
|
||||
assert "file_b.py" in paths
|
||||
assert "README.md" in paths
|
||||
|
||||
# All operations should be 'create'
|
||||
for entry in cs.entries:
|
||||
assert entry.operation == "create", (
|
||||
f"Expected 'create', got '{entry.operation}' for {entry.path}"
|
||||
)
|
||||
|
||||
# Verify summary is a non-empty string describing 3 changes
|
||||
summary_str = cs.summary()
|
||||
assert "3" in summary_str, f"Expected '3' in summary: {summary_str}"
|
||||
assert "create" in summary_str, f"Expected 'create' in summary: {summary_str}"
|
||||
|
||||
# Also verify via the domain-level SpecChangeSet summary (returns dict)
|
||||
spec_cs = capture.to_spec_changeset()
|
||||
spec_summary = spec_cs.summary()
|
||||
assert spec_summary["total"] == 3
|
||||
assert spec_summary["creates"] == 3
|
||||
|
||||
print("m2-changeset-multifile-ok")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
_COMMANDS: dict[str, object] = {
|
||||
"actor-yaml-create-load": actor_yaml_create_load,
|
||||
"actor-add-config-cli": actor_add_config_cli,
|
||||
"action-create": action_create,
|
||||
"plan-use-execute": plan_use_execute,
|
||||
"actor-yaml-parse-validate": actor_yaml_parse_validate,
|
||||
"actor-compile-stategraph": actor_compile_stategraph,
|
||||
"tool-router-resolve-external": tool_router_resolve_external,
|
||||
"validation-runner-execute": validation_runner_execute,
|
||||
"changeset-multifile": changeset_multifile,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
|
||||
sys.exit(1)
|
||||
fn = _COMMANDS[sys.argv[1]]
|
||||
fn() # type: ignore[operator]
|
||||
@@ -0,0 +1,73 @@
|
||||
*** Settings ***
|
||||
Documentation M1 end-to-end verification: action creation, resource registration, project linking, plan lifecycle, SQLite persistence, ChangeSet from tool invocations, git worktree sandbox isolation, and post-apply commit
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m1_e2e_verification.py
|
||||
|
||||
*** Test Cases ***
|
||||
Action Create From YAML Config
|
||||
[Documentation] Create an action from a YAML config file via agents action create --config
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-create cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-action-create-ok
|
||||
|
||||
Register Git Checkout Resource
|
||||
[Documentation] Register a git-checkout resource via agents resource add
|
||||
${result}= Run Process ${PYTHON} ${HELPER} resource-register cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-resource-register-ok
|
||||
|
||||
Create Project And Link Resource
|
||||
[Documentation] Create a project and link a resource to it
|
||||
${result}= Run Process ${PYTHON} ${HELPER} project-create-link cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-project-create-link-ok
|
||||
|
||||
Full Plan Lifecycle Use Execute Diff Apply
|
||||
[Documentation] Run full plan lifecycle: plan use -> plan execute -> plan diff -> plan apply
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-lifecycle cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-lifecycle-ok
|
||||
|
||||
SQLite Persistence For Plan And Action Records
|
||||
[Documentation] Verify Plan and Action records persist to SQLite with correct serialisation
|
||||
${result}= Run Process ${PYTHON} ${HELPER} sqlite-persistence cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-sqlite-persistence-ok
|
||||
|
||||
ChangeSet Built From Tool Invocations
|
||||
[Documentation] Verify ChangeSet is built from tool invocations, not parsed CLI output
|
||||
${result}= Run Process ${PYTHON} ${HELPER} changeset-invocations cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-changeset-invocations-ok
|
||||
|
||||
Git Worktree Sandbox Isolation
|
||||
[Documentation] Verify git worktree sandbox creates isolated directory and changes do not affect original until apply
|
||||
${result}= Run Process ${PYTHON} ${HELPER} sandbox-isolation cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-sandbox-isolation-ok
|
||||
|
||||
Post Apply Commit Exists In Target Repo
|
||||
[Documentation] Verify post-apply commit exists in the target repo after sandbox commit
|
||||
${result}= Run Process ${PYTHON} ${HELPER} post-apply-commit cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-post-apply-commit-ok
|
||||
@@ -0,0 +1,73 @@
|
||||
*** Settings ***
|
||||
Documentation M1 source-code plan lifecycle E2E smoke tests via CLI
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m1_sourcecode_smoke.py
|
||||
|
||||
*** Test Cases ***
|
||||
M1 Action Create From Config
|
||||
[Documentation] Create an action from M1 fixture YAML config
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-create cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-action-create-ok
|
||||
|
||||
M1 Plan Use Creates Strategize Plan
|
||||
[Documentation] Use an action to create a plan in strategize phase
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-ok
|
||||
|
||||
M1 Plan Use With Project Link
|
||||
[Documentation] Use action with a project argument to validate project linking
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use-project cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-project-ok
|
||||
|
||||
M1 Plan Execute Transitions Phase
|
||||
[Documentation] Execute a plan and verify phase transitions
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-execute cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-execute-ok
|
||||
|
||||
M1 Plan Diff Shows Changeset
|
||||
[Documentation] Run plan diff to show changeset
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-diff cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-diff-ok
|
||||
|
||||
M1 Plan Apply Reaches Terminal
|
||||
[Documentation] Apply a plan and verify terminal state
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-apply cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-apply-ok
|
||||
|
||||
M1 Full Lifecycle Action To Apply
|
||||
[Documentation] End-to-end: action create -> plan use -> execute -> apply
|
||||
${result}= Run Process ${PYTHON} ${HELPER} full-lifecycle cwd=${WORKSPACE} timeout=60s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-full-lifecycle-ok
|
||||
|
||||
M1 Plan Use With Plain Format
|
||||
[Documentation] Verify plan use --format plain stabilises assertions
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use-plain cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m1-plan-use-plain-ok
|
||||
@@ -0,0 +1,68 @@
|
||||
*** Settings ***
|
||||
Documentation M2 actor + tool source CLI smoke suite
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m2_actor_tool_smoke.py
|
||||
|
||||
*** Test Cases ***
|
||||
Actor Load Hierarchical YAML Fixture
|
||||
[Documentation] Load hierarchical actor from M2 fixture YAML and verify
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-load-fixture cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} actor-name: m2test/hierarchical-workflow
|
||||
Should Contain ${result.stdout} actor-type: graph
|
||||
Should Contain ${result.stdout} node-count: 3
|
||||
|
||||
Actor Discovery From Fixture Directory
|
||||
[Documentation] Discover actors from the M2 fixtures directory
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-discover cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} actor-count: 1
|
||||
Should Contain ${result.stdout} actor-loaded: m2test/hierarchical-workflow
|
||||
|
||||
Skill Pack Load From YAML Fixture
|
||||
[Documentation] Load skill pack from M2 fixture YAML and verify
|
||||
${result}= Run Process ${PYTHON} ${HELPER} skill-load-fixture cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} skill-name: m2test/file-ops-pack
|
||||
Should Contain ${result.stdout} tool-ref-count: 2
|
||||
Should Contain ${result.stdout} inline-tool-count: 1
|
||||
|
||||
Skill Registry And Tool Resolution
|
||||
[Documentation] Register skill pack and verify tool resolution
|
||||
${result}= Run Process ${PYTHON} ${HELPER} skill-registry cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} skill-registered: m2test/file-ops-pack
|
||||
Should Contain ${result.stdout} resolved-tools-ok
|
||||
|
||||
Tool Lifecycle Smoke
|
||||
[Documentation] Verify discover/activate/execute/deactivate lifecycle
|
||||
${result}= Run Process ${PYTHON} ${HELPER} tool-lifecycle cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} discover-ok
|
||||
Should Contain ${result.stdout} activate-ok
|
||||
Should Contain ${result.stdout} execute-ok
|
||||
Should Contain ${result.stdout} deactivate-ok
|
||||
|
||||
MCP Stub Discovery And Invocation
|
||||
[Documentation] Start MCP stub, discover tools, invoke, and stop
|
||||
${result}= Run Process ${PYTHON} ${HELPER} mcp-stub cwd=${WORKSPACE} timeout=30s
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} mcp-tool-count: 3
|
||||
Should Contain ${result.stdout} mcp-invoke-ok: mcp/search
|
||||
Should Contain ${result.stdout} mcp-stopped
|
||||
@@ -0,0 +1,103 @@
|
||||
*** Settings ***
|
||||
Documentation End-to-end verification of M2 success criteria:
|
||||
... actor compiler, tool routing, validation runner,
|
||||
... and multi-file ChangeSet generation.
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_m2_e2e_verification.py
|
||||
|
||||
*** Test Cases ***
|
||||
Actor YAML Create And Load Via Loader
|
||||
[Documentation] Create an actor YAML file and load it via ActorLoader.
|
||||
... Verifies that the YAML parses correctly and the actor
|
||||
... is discovered with the expected name and type.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-yaml-create-load cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-actor-yaml-create-load-ok
|
||||
|
||||
Actor Add Via Config CLI
|
||||
[Documentation] Create an actor YAML config and load it via the
|
||||
... ``agents actor add --config`` CLI command. Verifies
|
||||
... the actor is registered with the expected attributes.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-add-config-cli cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-actor-add-config-cli-ok
|
||||
|
||||
Action Create From Config Via CLI
|
||||
[Documentation] Create an action referencing a custom actor via
|
||||
... ``agents action create --config``. Verifies the
|
||||
... action is created with the correct name and actors.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} action-create cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-action-create-ok
|
||||
|
||||
Plan Use And Execute With Custom Actor
|
||||
[Documentation] Run ``plan use`` and ``plan execute`` with a custom
|
||||
... actor. Verifies the plan transitions through the
|
||||
... expected phases (strategize -> execute).
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-use-execute cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-plan-use-execute-ok
|
||||
|
||||
Actor YAML Parse And Validate
|
||||
[Documentation] Verify actor YAML files parse and validate correctly.
|
||||
... Tests valid graph/LLM YAML and confirms invalid YAML
|
||||
... is rejected with appropriate errors.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-yaml-parse-validate cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-actor-yaml-parse-validate-ok
|
||||
|
||||
Actor Compile To LangGraph StateGraph
|
||||
[Documentation] Verify actors compile to LangGraph StateGraphs.
|
||||
... Compiles a graph actor from examples and asserts
|
||||
... nodes, edges, entry point, and metadata. Also
|
||||
... confirms LLM actors are rejected by the compiler.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} actor-compile-stategraph cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-actor-compile-stategraph-ok
|
||||
|
||||
Tool Router Resolves External Tools
|
||||
[Documentation] Verify the tool router can resolve external tools
|
||||
... across OpenAI and Anthropic provider formats.
|
||||
... Tests format detection, normalization, and routing.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} tool-router-resolve-external cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-tool-router-resolve-external-ok
|
||||
|
||||
Validation Runner Executes Required And Informational
|
||||
[Documentation] Verify the validation runner executes both required
|
||||
... and informational validations correctly. Required
|
||||
... validations must pass; informational failures are
|
||||
... reported but do not block execution.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} validation-runner-execute cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-validation-runner-execute-ok
|
||||
|
||||
Multi File Generation Produces Correct ChangeSet
|
||||
[Documentation] Verify multi-file generation produces a correct
|
||||
... ChangeSet with the expected number of entries,
|
||||
... operations, and summary counts.
|
||||
${result}= Run Process ${PYTHON} ${HELPER} changeset-multifile cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} m2-changeset-multifile-ok
|
||||
@@ -560,7 +560,7 @@ Create Concurrent Retry Test Script
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}max_time = max(all_second_attempts)
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}spread = max_time - min_time
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}print(f"Retry time spread: {spread:.3f}s")
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}if spread > 0.01: # At least 10ms spread (jitter is working)
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}if spread > 0.001: # At least 1ms spread (jitter is working)
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}print("Jitter preventing thundering herd: SUCCESS")
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}sys.exit(0)
|
||||
... ${SPACE}${SPACE}${SPACE}${SPACE}else:
|
||||
|
||||
Reference in New Issue
Block a user