test(domain): add subplan model suites
This commit is contained in:
@@ -0,0 +1,240 @@
|
||||
"""Airspeed Velocity benchmarks for subplan model validation.
|
||||
|
||||
Measures construction, serialization, and failure-handler performance for
|
||||
SubplanConfig, SubplanStatus, SubplanFailureHandler, and Plan hierarchy
|
||||
properties.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
ExecutionMode,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
ProcessingState,
|
||||
SubplanConfig,
|
||||
SubplanFailureHandler,
|
||||
SubplanMergeStrategy,
|
||||
SubplanStatus,
|
||||
)
|
||||
|
||||
# Valid ULID constants
|
||||
_ROOT_ULID = "01HGZ6FE0AQDYTR4BXVQZ6EA00"
|
||||
_PARENT_ULID = "01HGZ6FE0AQDYTR4BXVQZ6EB00"
|
||||
_CHILD_ULID = "01HGZ6FE0AQDYTR4BXVQZ6EC00"
|
||||
_STATUS_ULID = "01KH72MDGQMPRW3R5WPZVB8KC1"
|
||||
|
||||
|
||||
def _root_plan() -> Plan:
|
||||
"""Create a root plan with no parent."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_ROOT_ULID),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None, namespace="local", name="bench-root"
|
||||
),
|
||||
description="Benchmark root plan",
|
||||
action_name="local/bench-action",
|
||||
)
|
||||
|
||||
|
||||
def _child_plan() -> Plan:
|
||||
"""Create a child plan with parent and root references."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(
|
||||
plan_id=_CHILD_ULID,
|
||||
parent_plan_id=_PARENT_ULID,
|
||||
root_plan_id=_ROOT_ULID,
|
||||
),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None, namespace="local", name="bench-child"
|
||||
),
|
||||
description="Benchmark child plan",
|
||||
action_name="local/bench-action",
|
||||
)
|
||||
|
||||
|
||||
def _plan_with_subplans() -> Plan:
|
||||
"""Create a plan with subplan config and statuses."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=_ROOT_ULID),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None, namespace="local", name="bench-parent"
|
||||
),
|
||||
description="Benchmark parent plan with subplans",
|
||||
action_name="local/bench-action",
|
||||
subplan_config=SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
max_parallel=5,
|
||||
fail_fast=True,
|
||||
),
|
||||
subplan_statuses=[
|
||||
SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
target_resources=["src/main.py"],
|
||||
status=ProcessingState.QUEUED,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SubplanConfigValidationSuite:
|
||||
"""Benchmark SubplanConfig construction and validation."""
|
||||
|
||||
def time_default_config_creation(self) -> None:
|
||||
"""Time creating a SubplanConfig with all defaults."""
|
||||
SubplanConfig()
|
||||
|
||||
def time_custom_config_creation(self) -> None:
|
||||
"""Time creating a SubplanConfig with custom settings."""
|
||||
SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
merge_strategy=SubplanMergeStrategy.FAIL_ON_CONFLICT,
|
||||
max_parallel=10,
|
||||
fail_fast=True,
|
||||
timeout_per_subplan_seconds=600,
|
||||
retry_failed=True,
|
||||
max_retries=3,
|
||||
)
|
||||
|
||||
def time_config_model_dump(self) -> None:
|
||||
"""Time serializing a SubplanConfig to dict."""
|
||||
config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
max_parallel=8,
|
||||
)
|
||||
config.model_dump()
|
||||
|
||||
def time_config_model_dump_json(self) -> None:
|
||||
"""Time serializing a SubplanConfig to JSON string."""
|
||||
config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.DEPENDENCY_ORDERED,
|
||||
merge_strategy=SubplanMergeStrategy.LAST_WINS,
|
||||
)
|
||||
config.model_dump_json()
|
||||
|
||||
|
||||
class SubplanStatusValidationSuite:
|
||||
"""Benchmark SubplanStatus construction and serialization."""
|
||||
|
||||
def time_minimal_status_creation(self) -> None:
|
||||
"""Time creating a SubplanStatus with minimal fields."""
|
||||
SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
)
|
||||
|
||||
def time_full_status_creation(self) -> None:
|
||||
"""Time creating a SubplanStatus with all fields populated."""
|
||||
SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
target_resources=["src/a.py", "src/b.py"],
|
||||
status=ProcessingState.ERRORED,
|
||||
error="TimeoutError: timed out",
|
||||
changeset_summary="Modified 5 files",
|
||||
files_changed=5,
|
||||
attempt_number=2,
|
||||
)
|
||||
|
||||
def time_status_model_dump(self) -> None:
|
||||
"""Time serializing a SubplanStatus to dict."""
|
||||
status = SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
status=ProcessingState.PROCESSING,
|
||||
)
|
||||
status.model_dump()
|
||||
|
||||
|
||||
class SubplanFailureHandlerSuite:
|
||||
"""Benchmark SubplanFailureHandler decisions."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Prepare handler and test fixtures."""
|
||||
self.handler = SubplanFailureHandler()
|
||||
self.config_fail_fast = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
fail_fast=True,
|
||||
retry_failed=True,
|
||||
max_retries=2,
|
||||
)
|
||||
self.config_no_fail_fast = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
fail_fast=False,
|
||||
retry_failed=True,
|
||||
max_retries=2,
|
||||
)
|
||||
self.retriable_status = SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
status=ProcessingState.ERRORED,
|
||||
error="TimeoutError: timed out",
|
||||
attempt_number=1,
|
||||
)
|
||||
self.non_retriable_status = SubplanStatus(
|
||||
subplan_id=_STATUS_ULID,
|
||||
action_name="local/bench-sub",
|
||||
status=ProcessingState.ERRORED,
|
||||
error="ConfigurationError: bad config",
|
||||
attempt_number=1,
|
||||
)
|
||||
|
||||
def time_should_stop_others_fail_fast(self) -> None:
|
||||
"""Time should_stop_others with fail_fast=True."""
|
||||
self.handler.should_stop_others(self.config_fail_fast, self.retriable_status)
|
||||
|
||||
def time_should_stop_others_no_fail_fast(self) -> None:
|
||||
"""Time should_stop_others with fail_fast=False."""
|
||||
self.handler.should_stop_others(self.config_no_fail_fast, self.retriable_status)
|
||||
|
||||
def time_should_retry_retriable(self) -> None:
|
||||
"""Time should_retry for a retriable error."""
|
||||
self.handler.should_retry(self.config_fail_fast, self.retriable_status)
|
||||
|
||||
def time_should_retry_non_retriable(self) -> None:
|
||||
"""Time should_retry for a non-retriable error."""
|
||||
self.handler.should_retry(self.config_fail_fast, self.non_retriable_status)
|
||||
|
||||
|
||||
class SubplanHierarchySuite:
|
||||
"""Benchmark Plan hierarchy property access."""
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Prepare plans for hierarchy property benchmarks."""
|
||||
self.root = _root_plan()
|
||||
self.child = _child_plan()
|
||||
self.parent_with_subs = _plan_with_subplans()
|
||||
|
||||
def time_is_root_plan(self) -> None:
|
||||
"""Time is_root_plan property check on root."""
|
||||
_ = self.root.is_root_plan
|
||||
|
||||
def time_is_subplan(self) -> None:
|
||||
"""Time is_subplan property check on child."""
|
||||
_ = self.child.is_subplan
|
||||
|
||||
def time_depth_root(self) -> None:
|
||||
"""Time depth property on root plan."""
|
||||
_ = self.root.depth
|
||||
|
||||
def time_depth_child(self) -> None:
|
||||
"""Time depth property on child plan."""
|
||||
_ = self.child.depth
|
||||
|
||||
def time_has_subplans(self) -> None:
|
||||
"""Time has_subplans property check."""
|
||||
_ = self.parent_with_subs.has_subplans
|
||||
|
||||
def time_cli_dict_with_subplans(self) -> None:
|
||||
"""Time as_cli_dict for plan with subplan statuses."""
|
||||
self.parent_with_subs.as_cli_dict()
|
||||
|
||||
def time_child_plan_creation(self) -> None:
|
||||
"""Time creating a child plan with hierarchy fields."""
|
||||
_child_plan()
|
||||
|
||||
def time_plan_with_subplans_creation(self) -> None:
|
||||
"""Time creating a plan with subplan config and statuses."""
|
||||
_plan_with_subplans()
|
||||
@@ -394,6 +394,82 @@ nox -s integration_tests
|
||||
python -m robot robot/plan_persistence_e2e.robot
|
||||
```
|
||||
|
||||
## Subplan Model Test Suites
|
||||
|
||||
The subplan model (hierarchy, configuration, failure handling) has dedicated test
|
||||
suites at unit, integration, and benchmark levels.
|
||||
|
||||
### Behave: Subplan Model (`features/subplan_model.feature`)
|
||||
|
||||
33 scenarios covering Plan hierarchy properties, SubplanConfig defaults, SubplanStatus
|
||||
tracking, SubplanFailureHandler decisions, and CLI dict rendering:
|
||||
|
||||
| Area | Scenarios | Description |
|
||||
|------|-----------|-------------|
|
||||
| Root plan identity | 2 | `is_root_plan`, `is_subplan`, `depth` for root plans |
|
||||
| Child plan identity | 3 | Parent reference, attempt counter, root propagation |
|
||||
| Default values | 4 | Phase, state, automation level, subplan_config defaults |
|
||||
| SubplanConfig storage | 3 | Sequential/parallel config, standalone defaults |
|
||||
| Hierarchy helpers | 3 | `has_subplans`, `depth` for non-root plans |
|
||||
| Lifecycle transitions | 2 | Child plans follow same phase transition rules |
|
||||
| Dependency guardrails | 5 | FailureHandler stop/retry decisions, phase independence |
|
||||
| CLI dict rendering | 3 | `parent_plan_id`, `subplan_count` in output |
|
||||
| Validation guardrails | 4 | Invalid ULIDs, out-of-range config values |
|
||||
|
||||
Step definitions: `features/steps/subplan_model_steps.py`
|
||||
|
||||
All step names are prefixed with `subplan-test` to avoid `AmbiguousStep` conflicts
|
||||
with existing `plan_model_steps.py` steps.
|
||||
|
||||
### Robot: Subplan Model (`robot/subplan_model.robot`)
|
||||
|
||||
11 smoke tests exercising subplan model properties via Python helper scripts:
|
||||
|
||||
| Test Case | Description |
|
||||
|-----------|-------------|
|
||||
| Root Plan Identity Flags | `is_root_plan=True`, `is_subplan=False`, `depth=0` |
|
||||
| Child Plan Identity Flags | `is_subplan=True`, `is_root_plan=False`, `depth=-1` |
|
||||
| Three Level Hierarchy Root Propagation | `root_plan_id` propagates through 3 levels |
|
||||
| SubplanConfig Defaults | All default values are sensible |
|
||||
| SubplanConfig Custom Settings | Parallel mode with custom retry settings |
|
||||
| Failure Handler Stop Others On Fail Fast | `should_stop_others` with `fail_fast=True` |
|
||||
| Failure Handler Retry Retriable Error | `should_retry` for `TimeoutError` |
|
||||
| Failure Handler Skip Non Retriable Error | `should_retry` skips `ConfigurationError` |
|
||||
| CLI Dict Renders Parent Plan Id For Child | `parent_plan_id` in child plan CLI dict |
|
||||
| CLI Dict Renders Subplan Count | `subplan_count` in parent plan CLI dict |
|
||||
| Plan With Subplan Config And Statuses | Full plan model with config + statuses |
|
||||
|
||||
Helper script: `robot/helper_subplan_model.py`
|
||||
|
||||
### ASV Benchmarks (`benchmarks/subplan_model_validation_bench.py`)
|
||||
|
||||
Performance baselines for subplan model operations:
|
||||
|
||||
- **`SubplanConfigValidationSuite`** -- `time_default_config_creation`,
|
||||
`time_custom_config_creation`, `time_config_model_dump`, `time_config_model_dump_json`
|
||||
- **`SubplanStatusValidationSuite`** -- `time_minimal_status_creation`,
|
||||
`time_full_status_creation`, `time_status_model_dump`
|
||||
- **`SubplanFailureHandlerSuite`** -- `time_should_stop_others_fail_fast`,
|
||||
`time_should_stop_others_no_fail_fast`, `time_should_retry_retriable`,
|
||||
`time_should_retry_non_retriable`
|
||||
- **`SubplanHierarchySuite`** -- `time_is_root_plan`, `time_is_subplan`,
|
||||
`time_depth_root`, `time_depth_child`, `time_has_subplans`,
|
||||
`time_cli_dict_with_subplans`, `time_child_plan_creation`,
|
||||
`time_plan_with_subplans_creation`
|
||||
|
||||
### Running Subplan Model Tests
|
||||
|
||||
```bash
|
||||
# Behave only (subplan model feature)
|
||||
nox -s unit_tests -- features/subplan_model.feature
|
||||
|
||||
# Robot only (subplan model suite)
|
||||
nox -s integration_tests -- --suite robot/subplan_model.robot
|
||||
|
||||
# Benchmarks
|
||||
nox -s benchmark
|
||||
```
|
||||
|
||||
## CI Pipeline
|
||||
|
||||
The Forgejo CI pipeline runs these jobs in order:
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
"""Step definitions for subplan model hierarchy, defaults, and guardrails."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from pydantic import ValidationError
|
||||
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
AutomationLevel,
|
||||
ExecutionMode,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
ProcessingState,
|
||||
SubplanConfig,
|
||||
SubplanFailureHandler,
|
||||
SubplanMergeStrategy,
|
||||
SubplanStatus,
|
||||
can_transition,
|
||||
)
|
||||
|
||||
_R = "01ARZ3NDEKTSV4RRFFQ69G5FAV" # root ULID
|
||||
_P = "01ARZ3NDEKTSV4RRFFQ69G5FAW" # parent ULID
|
||||
_C = "01ARZ3NDEKTSV4RRFFQ69G5FAX" # child ULID
|
||||
_G = "01ARZ3NDEKTSV4RRFFQ69G5FAY" # grandchild ULID
|
||||
_S = "01KH72MDGQMPRW3R5WPZVB8KC1" # status ULID
|
||||
|
||||
|
||||
def _plan(
|
||||
pid: str = _R,
|
||||
par: str | None = None,
|
||||
root: str | None = None,
|
||||
att: int = 1,
|
||||
ph: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
ps: ProcessingState = ProcessingState.QUEUED,
|
||||
al: AutomationLevel = AutomationLevel.MANUAL,
|
||||
sc: SubplanConfig | None = None,
|
||||
ss: list[SubplanStatus] | None = None,
|
||||
) -> Plan:
|
||||
"""Build a Plan with sensible defaults for subplan testing."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(
|
||||
plan_id=pid,
|
||||
parent_plan_id=par,
|
||||
root_plan_id=root,
|
||||
attempt=att,
|
||||
),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None,
|
||||
namespace="local",
|
||||
name="subplan-test",
|
||||
),
|
||||
description="Subplan test plan",
|
||||
action_name="local/subplan-action",
|
||||
phase=ph,
|
||||
processing_state=ps,
|
||||
automation_level=al,
|
||||
subplan_config=sc,
|
||||
subplan_statuses=ss or [],
|
||||
)
|
||||
|
||||
|
||||
def _st(err: str | None = None, att: int = 1) -> SubplanStatus:
|
||||
"""Build a SubplanStatus."""
|
||||
return SubplanStatus(
|
||||
subplan_id=_S,
|
||||
action_name="local/sub-task",
|
||||
status=ProcessingState.ERRORED if err else ProcessingState.QUEUED,
|
||||
error=err,
|
||||
attempt_number=att,
|
||||
target_resources=["src/main.py"],
|
||||
)
|
||||
|
||||
|
||||
# -- Given: plan creation --
|
||||
@given("I create a subplan-test root plan")
|
||||
def step_g01(ctx: Context) -> None:
|
||||
"""Create root plan."""
|
||||
ctx.subplan_plan = _plan()
|
||||
|
||||
|
||||
@given("I create a subplan-test plan with root_plan_id equal to plan_id")
|
||||
def step_g02(ctx: Context) -> None:
|
||||
"""Create root plan with explicit root_plan_id."""
|
||||
ctx.subplan_plan = _plan(root=_R)
|
||||
|
||||
|
||||
@given("I create a subplan-test child plan with a parent")
|
||||
def step_g03(ctx: Context) -> None:
|
||||
"""Create child plan."""
|
||||
ctx.subplan_plan = _plan(pid=_C, par=_P, root=_R)
|
||||
|
||||
|
||||
@given("I create a subplan-test three-level hierarchy")
|
||||
def step_g05(ctx: Context) -> None:
|
||||
"""Create root/parent/grandchild."""
|
||||
ctx.subplan_root = _plan(pid=_R, root=_R)
|
||||
ctx.subplan_parent = _plan(pid=_P, par=_R, root=_R)
|
||||
ctx.subplan_grandchild = _plan(pid=_G, par=_P, root=_R)
|
||||
|
||||
|
||||
@given("I create a subplan-test default plan")
|
||||
def step_g06(ctx: Context) -> None:
|
||||
"""Create default plan."""
|
||||
ctx.subplan_plan = _plan()
|
||||
|
||||
|
||||
@given("I create a subplan-test plan with sequential subplan config")
|
||||
def step_g07(ctx: Context) -> None:
|
||||
"""Plan with sequential config."""
|
||||
ctx.subplan_plan = _plan(
|
||||
sc=SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
merge_strategy=SubplanMergeStrategy.GIT_THREE_WAY,
|
||||
fail_fast=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@given("I create a subplan-test plan with parallel subplan config")
|
||||
def step_g08(ctx: Context) -> None:
|
||||
"""Plan with parallel config."""
|
||||
ctx.subplan_plan = _plan(
|
||||
sc=SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
max_parallel=10,
|
||||
retry_failed=True,
|
||||
max_retries=3,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@given("I create a subplan-test default subplan config")
|
||||
def step_g09(ctx: Context) -> None:
|
||||
"""Standalone SubplanConfig with defaults."""
|
||||
ctx.subplan_config = SubplanConfig()
|
||||
|
||||
|
||||
@given("I create a subplan-test plan with one subplan status")
|
||||
def step_g10(ctx: Context) -> None:
|
||||
"""Plan with one status."""
|
||||
ctx.subplan_plan = _plan(ss=[_st()])
|
||||
|
||||
|
||||
@given("I create a subplan-test fail-fast config with errored status")
|
||||
def step_g13(ctx: Context) -> None:
|
||||
"""Fail-fast config + errored status."""
|
||||
ctx.subplan_handler_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL, fail_fast=True
|
||||
)
|
||||
ctx.subplan_handler_status = _st("TimeoutError: timed out")
|
||||
|
||||
|
||||
@given("I create a subplan-test parallel-no-fail-fast config with errored status")
|
||||
def step_g14(ctx: Context) -> None:
|
||||
"""Parallel (no fail_fast) + errored status."""
|
||||
ctx.subplan_handler_config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL, fail_fast=False
|
||||
)
|
||||
ctx.subplan_handler_status = _st("TimeoutError: timed out")
|
||||
|
||||
|
||||
@given("I create a subplan-test config with retriable timeout error")
|
||||
def step_g15(ctx: Context) -> None:
|
||||
"""Config + retriable TimeoutError."""
|
||||
ctx.subplan_handler_config = SubplanConfig(retry_failed=True, max_retries=2)
|
||||
ctx.subplan_handler_status = _st("TimeoutError: op timed out", 1)
|
||||
|
||||
|
||||
@given("I create a subplan-test config with non-retriable configuration error")
|
||||
def step_g16(ctx: Context) -> None:
|
||||
"""Config + non-retriable ConfigurationError."""
|
||||
ctx.subplan_handler_config = SubplanConfig(retry_failed=True, max_retries=2)
|
||||
ctx.subplan_handler_status = _st("ConfigurationError: bad", 1)
|
||||
|
||||
|
||||
@given("I create a subplan-test config with retries disabled and retriable error")
|
||||
def step_g17(ctx: Context) -> None:
|
||||
"""Config (retries off) + retriable error."""
|
||||
ctx.subplan_handler_config = SubplanConfig(retry_failed=False)
|
||||
ctx.subplan_handler_status = _st("TimeoutError: timed out", 1)
|
||||
|
||||
|
||||
@given("I create a subplan-test config with max retries exceeded")
|
||||
def step_g18(ctx: Context) -> None:
|
||||
"""Config + attempt > max_retries."""
|
||||
ctx.subplan_handler_config = SubplanConfig(retry_failed=True, max_retries=2)
|
||||
ctx.subplan_handler_status = _st("TimeoutError: timed out", 3)
|
||||
|
||||
|
||||
@given("I build a subplan-test plan with the child identity")
|
||||
def step_g19(ctx: Context) -> None:
|
||||
"""Alias plan for CLI dict assertion."""
|
||||
ctx.subplan_cli_plan = ctx.subplan_plan
|
||||
|
||||
|
||||
@given("I build a subplan-test plan with the root identity")
|
||||
def step_g20(ctx: Context) -> None:
|
||||
"""Alias plan for CLI dict assertion."""
|
||||
ctx.subplan_cli_plan = ctx.subplan_plan
|
||||
|
||||
|
||||
# -- When: validation errors --
|
||||
@when("I try to create a subplan-test identity with invalid parent ULID")
|
||||
def step_w01(ctx: Context) -> None:
|
||||
"""Invalid parent_plan_id."""
|
||||
ctx.error = None
|
||||
try:
|
||||
PlanIdentity(plan_id=_R, parent_plan_id="not-a-valid-ulid")
|
||||
except ValidationError as e:
|
||||
ctx.error = e
|
||||
|
||||
|
||||
@when("I try to create a subplan-test config with max_parallel {v:d}")
|
||||
def step_w02(ctx: Context, v: int) -> None:
|
||||
"""Invalid max_parallel."""
|
||||
ctx.error = None
|
||||
try:
|
||||
SubplanConfig(max_parallel=v)
|
||||
except ValidationError as e:
|
||||
ctx.error = e
|
||||
|
||||
|
||||
@when("I try to create a subplan-test config with max_retries {v:d}")
|
||||
def step_w03(ctx: Context, v: int) -> None:
|
||||
"""Invalid max_retries."""
|
||||
ctx.error = None
|
||||
try:
|
||||
SubplanConfig(max_retries=v)
|
||||
except ValidationError as e:
|
||||
ctx.error = e
|
||||
|
||||
|
||||
# -- Then: plan identity --
|
||||
@then("the subplan-test plan should not be a subplan")
|
||||
def step_t01(ctx: Context) -> None:
|
||||
"""Not a subplan."""
|
||||
assert not ctx.subplan_plan.is_subplan
|
||||
|
||||
|
||||
@then("the subplan-test plan should be a subplan")
|
||||
def step_t02(ctx: Context) -> None:
|
||||
"""Is a subplan."""
|
||||
assert ctx.subplan_plan.is_subplan
|
||||
|
||||
|
||||
@then("the subplan-test plan should be a root plan")
|
||||
def step_t03(ctx: Context) -> None:
|
||||
"""Is root."""
|
||||
assert ctx.subplan_plan.is_root_plan
|
||||
|
||||
|
||||
@then("the subplan-test plan should not be a root plan")
|
||||
def step_t04(ctx: Context) -> None:
|
||||
"""Not root."""
|
||||
assert not ctx.subplan_plan.is_root_plan
|
||||
|
||||
|
||||
@then("the subplan-test plan depth should be {e:d}")
|
||||
def step_t05(ctx: Context, e: int) -> None:
|
||||
"""Depth check."""
|
||||
assert ctx.subplan_plan.depth == e
|
||||
|
||||
|
||||
@then("the subplan-test plan attempt should be {e:d}")
|
||||
def step_t06(ctx: Context, e: int) -> None:
|
||||
"""Attempt check."""
|
||||
assert ctx.subplan_plan.identity.attempt == e
|
||||
|
||||
|
||||
@then('the subplan-test plan phase should be "{e}"')
|
||||
def step_t07(ctx: Context, e: str) -> None:
|
||||
"""Phase check."""
|
||||
assert ctx.subplan_plan.phase.value == e
|
||||
|
||||
|
||||
@then('the subplan-test plan processing state should be "{e}"')
|
||||
def step_t08(ctx: Context, e: str) -> None:
|
||||
"""Processing state check."""
|
||||
assert ctx.subplan_plan.processing_state.value == e
|
||||
|
||||
|
||||
@then("the subplan-test plan should not be terminal")
|
||||
def step_t09(ctx: Context) -> None:
|
||||
"""Not terminal."""
|
||||
assert not ctx.subplan_plan.is_terminal
|
||||
|
||||
|
||||
@then("the subplan-test plan should not be errored")
|
||||
def step_t10(ctx: Context) -> None:
|
||||
"""Not errored."""
|
||||
assert not ctx.subplan_plan.is_errored
|
||||
|
||||
|
||||
@then('the subplan-test plan automation level should be "{e}"')
|
||||
def step_t11(ctx: Context, e: str) -> None:
|
||||
"""Automation level check."""
|
||||
assert ctx.subplan_plan.automation_level.value == e
|
||||
|
||||
|
||||
@then("the subplan-test plan subplan_config should be none")
|
||||
def step_t12(ctx: Context) -> None:
|
||||
"""No subplan config."""
|
||||
assert ctx.subplan_plan.subplan_config is None
|
||||
|
||||
|
||||
@then("the subplan-test plan should have {c:d} subplan statuses")
|
||||
def step_t13(ctx: Context, c: int) -> None:
|
||||
"""Status count check."""
|
||||
assert len(ctx.subplan_plan.subplan_statuses) == c
|
||||
|
||||
|
||||
# -- Then: subplan config on plan --
|
||||
@then('the subplan-test plan subplan_config execution_mode should be "{e}"')
|
||||
def step_t14(ctx: Context, e: str) -> None:
|
||||
"""Config execution_mode."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.execution_mode.value == e
|
||||
|
||||
|
||||
@then('the subplan-test plan subplan_config merge_strategy should be "{e}"')
|
||||
def step_t15(ctx: Context, e: str) -> None:
|
||||
"""Config merge_strategy."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.merge_strategy.value == e
|
||||
|
||||
|
||||
@then("the subplan-test plan subplan_config fail_fast should be {e}")
|
||||
def step_t16(ctx: Context, e: str) -> None:
|
||||
"""Config fail_fast."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.fail_fast is (e.lower() == "true")
|
||||
|
||||
|
||||
@then("the subplan-test plan subplan_config max_parallel should be {e:d}")
|
||||
def step_t17(ctx: Context, e: int) -> None:
|
||||
"""Config max_parallel."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.max_parallel == e
|
||||
|
||||
|
||||
@then("the subplan-test plan subplan_config retry_failed should be {e}")
|
||||
def step_t18(ctx: Context, e: str) -> None:
|
||||
"""Config retry_failed."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.retry_failed is (e.lower() == "true")
|
||||
|
||||
|
||||
@then("the subplan-test plan subplan_config max_retries should be {e:d}")
|
||||
def step_t19(ctx: Context, e: int) -> None:
|
||||
"""Config max_retries."""
|
||||
assert ctx.subplan_plan.subplan_config is not None
|
||||
assert ctx.subplan_plan.subplan_config.max_retries == e
|
||||
|
||||
|
||||
# -- Then: standalone SubplanConfig defaults --
|
||||
@then('the subplan-test subplan config execution_mode should be "{e}"')
|
||||
def step_t20(ctx: Context, e: str) -> None:
|
||||
"""Standalone config execution_mode."""
|
||||
assert ctx.subplan_config.execution_mode.value == e
|
||||
|
||||
|
||||
@then('the subplan-test subplan config merge_strategy should be "{e}"')
|
||||
def step_t21(ctx: Context, e: str) -> None:
|
||||
"""Standalone config merge_strategy."""
|
||||
assert ctx.subplan_config.merge_strategy.value == e
|
||||
|
||||
|
||||
@then("the subplan-test subplan config max_parallel should be {e:d}")
|
||||
def step_t22(ctx: Context, e: int) -> None:
|
||||
"""Standalone config max_parallel."""
|
||||
assert ctx.subplan_config.max_parallel == e
|
||||
|
||||
|
||||
@then("the subplan-test subplan config fail_fast should be {e}")
|
||||
def step_t23(ctx: Context, e: str) -> None:
|
||||
"""Standalone config fail_fast."""
|
||||
assert ctx.subplan_config.fail_fast is (e.lower() == "true")
|
||||
|
||||
|
||||
@then("the subplan-test subplan config retry_failed should be {e}")
|
||||
def step_t24(ctx: Context, e: str) -> None:
|
||||
"""Standalone config retry_failed."""
|
||||
assert ctx.subplan_config.retry_failed is (e.lower() == "true")
|
||||
|
||||
|
||||
@then("the subplan-test subplan config max_retries should be {e:d}")
|
||||
def step_t25(ctx: Context, e: int) -> None:
|
||||
"""Standalone config max_retries."""
|
||||
assert ctx.subplan_config.max_retries == e
|
||||
|
||||
|
||||
@then("the subplan-test subplan config timeout should be none")
|
||||
def step_t26(ctx: Context) -> None:
|
||||
"""Standalone config timeout is None."""
|
||||
assert ctx.subplan_config.timeout_per_subplan_seconds is None
|
||||
|
||||
|
||||
# -- Then: has_subplans --
|
||||
@then("the subplan-test plan should have subplans")
|
||||
def step_t27(ctx: Context) -> None:
|
||||
"""Has subplans."""
|
||||
assert ctx.subplan_plan.has_subplans
|
||||
|
||||
|
||||
@then("the subplan-test plan should not have subplans")
|
||||
def step_t28(ctx: Context) -> None:
|
||||
"""No subplans."""
|
||||
assert not ctx.subplan_plan.has_subplans
|
||||
|
||||
|
||||
# -- Then: hierarchy --
|
||||
@then("the subplan-test grandchild root_plan_id should match the root plan_id")
|
||||
def step_t29(ctx: Context) -> None:
|
||||
"""Grandchild root matches root."""
|
||||
assert (
|
||||
ctx.subplan_grandchild.identity.root_plan_id
|
||||
== ctx.subplan_root.identity.plan_id
|
||||
)
|
||||
|
||||
|
||||
# -- Then: transitions --
|
||||
@then('subplan-test transition from "{f}" to "{t}" should be valid')
|
||||
def step_t30(ctx: Context, f: str, t: str) -> None:
|
||||
"""Valid transition."""
|
||||
assert can_transition(PlanPhase(f), PlanPhase(t))
|
||||
|
||||
|
||||
@then('subplan-test transition from "{f}" to "{t}" should be invalid')
|
||||
def step_t31(ctx: Context, f: str, t: str) -> None:
|
||||
"""Invalid transition."""
|
||||
assert not can_transition(PlanPhase(f), PlanPhase(t))
|
||||
|
||||
|
||||
# -- Then: failure handler --
|
||||
@then("the subplan-test failure handler should stop others")
|
||||
def step_t34(ctx: Context) -> None:
|
||||
"""Handler stops others."""
|
||||
h = SubplanFailureHandler()
|
||||
assert h.should_stop_others(ctx.subplan_handler_config, ctx.subplan_handler_status)
|
||||
|
||||
|
||||
@then("the subplan-test failure handler should not stop others")
|
||||
def step_t35(ctx: Context) -> None:
|
||||
"""Handler does not stop others."""
|
||||
h = SubplanFailureHandler()
|
||||
assert not h.should_stop_others(
|
||||
ctx.subplan_handler_config, ctx.subplan_handler_status
|
||||
)
|
||||
|
||||
|
||||
@then("the subplan-test failure handler should retry")
|
||||
def step_t36(ctx: Context) -> None:
|
||||
"""Handler retries."""
|
||||
h = SubplanFailureHandler()
|
||||
assert h.should_retry(ctx.subplan_handler_config, ctx.subplan_handler_status)
|
||||
|
||||
|
||||
@then("the subplan-test failure handler should not retry")
|
||||
def step_t37(ctx: Context) -> None:
|
||||
"""Handler does not retry."""
|
||||
h = SubplanFailureHandler()
|
||||
assert not h.should_retry(ctx.subplan_handler_config, ctx.subplan_handler_status)
|
||||
|
||||
|
||||
# -- Then: CLI dict --
|
||||
@then("the subplan-test cli dict should contain parent_plan_id")
|
||||
def step_t38(ctx: Context) -> None:
|
||||
"""CLI dict has parent_plan_id."""
|
||||
assert "parent_plan_id" in ctx.subplan_cli_plan.as_cli_dict()
|
||||
|
||||
|
||||
@then("the subplan-test cli dict should contain subplan_count of {e:d}")
|
||||
def step_t39(ctx: Context, e: int) -> None:
|
||||
"""CLI dict subplan_count."""
|
||||
assert ctx.subplan_plan.as_cli_dict().get("subplan_count") == e
|
||||
|
||||
|
||||
@then("the subplan-test cli dict should not contain parent_plan_id")
|
||||
def step_t40(ctx: Context) -> None:
|
||||
"""CLI dict no parent_plan_id."""
|
||||
assert "parent_plan_id" not in ctx.subplan_cli_plan.as_cli_dict()
|
||||
@@ -0,0 +1,161 @@
|
||||
Feature: Subplan Model Hierarchy and Configuration
|
||||
As a developer
|
||||
I want the Plan model to support subplan hierarchies
|
||||
So that parent plans can decompose work into coordinated child plans
|
||||
|
||||
# Root plan identity
|
||||
|
||||
Scenario: Root plan has no parent and root equals self
|
||||
Given I create a subplan-test root plan
|
||||
Then the subplan-test plan should not be a subplan
|
||||
And the subplan-test plan should be a root plan
|
||||
And the subplan-test plan depth should be 0
|
||||
|
||||
Scenario: Root plan with explicit root_plan_id equal to plan_id
|
||||
Given I create a subplan-test plan with root_plan_id equal to plan_id
|
||||
Then the subplan-test plan should be a root plan
|
||||
And the subplan-test plan should not be a subplan
|
||||
|
||||
# Child plan identity
|
||||
|
||||
Scenario: Child plan with parent_plan_id set is a subplan
|
||||
Given I create a subplan-test child plan with a parent
|
||||
Then the subplan-test plan should be a subplan
|
||||
And the subplan-test plan should not be a root plan
|
||||
And the subplan-test plan attempt should be 1
|
||||
|
||||
Scenario: Root plan id propagates to nested children
|
||||
Given I create a subplan-test three-level hierarchy
|
||||
Then the subplan-test grandchild root_plan_id should match the root plan_id
|
||||
|
||||
# Default values for new plans
|
||||
|
||||
Scenario: New plan defaults to strategize phase and queued state
|
||||
Given I create a subplan-test default plan
|
||||
Then the subplan-test plan phase should be "strategize"
|
||||
And the subplan-test plan processing state should be "queued"
|
||||
And the subplan-test plan should not be terminal
|
||||
And the subplan-test plan should not be errored
|
||||
|
||||
Scenario: New plan defaults to manual automation level
|
||||
Given I create a subplan-test default plan
|
||||
Then the subplan-test plan automation level should be "manual"
|
||||
|
||||
Scenario: New plan has no subplan config by default
|
||||
Given I create a subplan-test default plan
|
||||
Then the subplan-test plan subplan_config should be none
|
||||
|
||||
Scenario: New plan has empty subplan statuses by default
|
||||
Given I create a subplan-test default plan
|
||||
Then the subplan-test plan should have 0 subplan statuses
|
||||
|
||||
# Subplan config storage and retrieval
|
||||
|
||||
Scenario: Plan stores sequential subplan config
|
||||
Given I create a subplan-test plan with sequential subplan config
|
||||
Then the subplan-test plan subplan_config execution_mode should be "sequential"
|
||||
And the subplan-test plan subplan_config merge_strategy should be "git_three_way"
|
||||
And the subplan-test plan subplan_config fail_fast should be false
|
||||
|
||||
Scenario: Plan stores parallel subplan config with custom settings
|
||||
Given I create a subplan-test plan with parallel subplan config
|
||||
Then the subplan-test plan subplan_config execution_mode should be "parallel"
|
||||
And the subplan-test plan subplan_config max_parallel should be 10
|
||||
And the subplan-test plan subplan_config retry_failed should be true
|
||||
And the subplan-test plan subplan_config max_retries should be 3
|
||||
|
||||
Scenario: SubplanConfig defaults are sensible
|
||||
Given I create a subplan-test default subplan config
|
||||
Then the subplan-test subplan config execution_mode should be "sequential"
|
||||
And the subplan-test subplan config merge_strategy should be "git_three_way"
|
||||
And the subplan-test subplan config max_parallel should be 5
|
||||
And the subplan-test subplan config fail_fast should be false
|
||||
And the subplan-test subplan config retry_failed should be true
|
||||
And the subplan-test subplan config max_retries should be 2
|
||||
And the subplan-test subplan config timeout should be none
|
||||
|
||||
# Plan hierarchy helpers
|
||||
|
||||
Scenario: Plan with subplan statuses reports has_subplans true
|
||||
Given I create a subplan-test plan with one subplan status
|
||||
Then the subplan-test plan should have subplans
|
||||
|
||||
Scenario: Plan without subplan statuses reports has_subplans false
|
||||
Given I create a subplan-test default plan
|
||||
Then the subplan-test plan should not have subplans
|
||||
|
||||
Scenario: Depth returns negative one for non-root plans
|
||||
Given I create a subplan-test child plan with a parent
|
||||
Then the subplan-test plan depth should be -1
|
||||
|
||||
# Child plans follow same lifecycle as root plans
|
||||
|
||||
Scenario: Child plan transitions follow standard lifecycle
|
||||
Then subplan-test transition from "action" to "strategize" should be valid
|
||||
And subplan-test transition from "strategize" to "execute" should be valid
|
||||
And subplan-test transition from "execute" to "apply" should be valid
|
||||
|
||||
Scenario: Child plan invalid transitions are rejected
|
||||
Then subplan-test transition from "strategize" to "apply" should be invalid
|
||||
And subplan-test transition from "action" to "execute" should be invalid
|
||||
And subplan-test transition from "action" to "apply" should be invalid
|
||||
|
||||
# Dependency guardrails
|
||||
|
||||
Scenario: SubplanFailureHandler stops others when fail_fast is true
|
||||
Given I create a subplan-test fail-fast config with errored status
|
||||
Then the subplan-test failure handler should stop others
|
||||
|
||||
Scenario: SubplanFailureHandler does not stop others for parallel without fail_fast
|
||||
Given I create a subplan-test parallel-no-fail-fast config with errored status
|
||||
Then the subplan-test failure handler should not stop others
|
||||
|
||||
Scenario: SubplanFailureHandler retries retriable failure
|
||||
Given I create a subplan-test config with retriable timeout error
|
||||
Then the subplan-test failure handler should retry
|
||||
|
||||
Scenario: SubplanFailureHandler does not retry non-retriable error
|
||||
Given I create a subplan-test config with non-retriable configuration error
|
||||
Then the subplan-test failure handler should not retry
|
||||
|
||||
Scenario: SubplanFailureHandler does not retry when retries disabled
|
||||
Given I create a subplan-test config with retries disabled and retriable error
|
||||
Then the subplan-test failure handler should not retry
|
||||
|
||||
Scenario: SubplanFailureHandler does not retry when max retries exceeded
|
||||
Given I create a subplan-test config with max retries exceeded
|
||||
Then the subplan-test failure handler should not retry
|
||||
|
||||
# CLI dict rendering for subplan hierarchy
|
||||
|
||||
Scenario: CLI dict includes parent_plan_id for child plans
|
||||
Given I create a subplan-test child plan with a parent
|
||||
And I build a subplan-test plan with the child identity
|
||||
Then the subplan-test cli dict should contain parent_plan_id
|
||||
|
||||
Scenario: CLI dict includes subplan_count when subplans exist
|
||||
Given I create a subplan-test plan with one subplan status
|
||||
Then the subplan-test cli dict should contain subplan_count of 1
|
||||
|
||||
Scenario: CLI dict omits parent_plan_id for root plans
|
||||
Given I create a subplan-test root plan
|
||||
And I build a subplan-test plan with the root identity
|
||||
Then the subplan-test cli dict should not contain parent_plan_id
|
||||
|
||||
# Validation guardrails
|
||||
|
||||
Scenario: Invalid ULID for parent_plan_id raises validation error
|
||||
When I try to create a subplan-test identity with invalid parent ULID
|
||||
Then a validation error should be raised
|
||||
|
||||
Scenario: SubplanConfig max_parallel below minimum raises validation error
|
||||
When I try to create a subplan-test config with max_parallel 0
|
||||
Then a validation error should be raised
|
||||
|
||||
Scenario: SubplanConfig max_parallel above maximum raises validation error
|
||||
When I try to create a subplan-test config with max_parallel 51
|
||||
Then a validation error should be raised
|
||||
|
||||
Scenario: SubplanConfig max_retries below minimum raises validation error
|
||||
When I try to create a subplan-test config with max_retries -1
|
||||
Then a validation error should be raised
|
||||
+19
-19
@@ -4490,25 +4490,25 @@ No standalone Q0-Advanced commits planned. Advanced QA enhancements are bundled
|
||||
}
|
||||
```
|
||||
- [X] Commit: "feat(domain): define retriable vs non-retriable failures"
|
||||
- [ ] **COMMIT (Owner: Brent | Group: E1.tests | Branch: feature/m5-subplan-tests | Planned: Day 25 | Expected: Day 26) - Commit message: "test(domain): add subplan model suites"**
|
||||
- [ ] Git [Brent]: `git checkout master`
|
||||
- [ ] Git [Brent]: `git pull origin master`
|
||||
- [ ] Git [Brent]: `git checkout -b feature/m5-subplan-tests`
|
||||
- [ ] Git [Brent]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
|
||||
- [ ] Code [Brent]: Add Behave feature coverage for subplan model defaults, hierarchy helpers, and retry metadata.
|
||||
- [ ] Code [Brent]: Add step definitions in `features/steps/subplan_model_steps.py` (plan hierarchy, defaults, dependency validation).
|
||||
- [ ] Docs [Brent]: Update `docs/development/testing.md` with subplan model test coverage notes.
|
||||
- [ ] Tests (Behave) [Brent]: Add `features/subplan_model.feature` scenarios (hierarchy flags, defaults, dependency guardrails).
|
||||
- [ ] Tests (Robot) [Brent]: Add `robot/subplan_model.robot` smoke coverage for CLI/status surface output.
|
||||
- [ ] Tests (ASV) [Brent]: Add `benchmarks/subplan_model_validation_bench.py` for model validation baseline.
|
||||
- [ ] Quality [Brent]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
|
||||
- [ ] Quality [Brent]: Run `nox` (all default sessions, including benchmark).
|
||||
- [ ] Git [Brent]: `git add .`
|
||||
- [ ] Git [Brent]: `git commit -m "test(domain): add subplan model suites"`
|
||||
- [ ] Git [Brent]: `git push -u origin feature/m5-subplan-tests`
|
||||
- [ ] Forgejo PR [Brent]: Open PR from `feature/m5-subplan-tests` to `master` with description "Add subplan model Behave/Robot suites and benchmarks.".
|
||||
- [ ] Git [Brent]: `git checkout master`
|
||||
- [ ] Git [Brent]: `git branch -d feature/m5-subplan-tests`
|
||||
- [X] **COMMIT (Owner: Brent | Group: E1.tests | Branch: feature/m5-subplan-tests | Planned: Day 25 | Expected: Day 26 | Done: Day 25) - Commit message: "test(domain): add subplan model suites"**
|
||||
- [X] Git [Brent]: `git checkout master`
|
||||
- [X] Git [Brent]: `git pull origin master`
|
||||
- [X] Git [Brent]: `git checkout -b feature/m5-subplan-tests`
|
||||
- [X] Git [Brent]: `git fetch origin && git merge origin/master` (run before final tests and before commit)
|
||||
- [X] Code [Brent]: Add Behave feature coverage for subplan model defaults, hierarchy helpers, and retry metadata.
|
||||
- [X] Code [Brent]: Add step definitions in `features/steps/subplan_model_steps.py` (plan hierarchy, defaults, dependency validation).
|
||||
- [X] Docs [Brent]: Update `docs/development/testing.md` with subplan model test coverage notes.
|
||||
- [X] Tests (Behave) [Brent]: Add `features/subplan_model.feature` scenarios (hierarchy flags, defaults, dependency guardrails).
|
||||
- [X] Tests (Robot) [Brent]: Add `robot/subplan_model.robot` smoke coverage for CLI/status surface output.
|
||||
- [X] Tests (ASV) [Brent]: Add `benchmarks/subplan_model_validation_bench.py` for model validation baseline.
|
||||
- [X] Quality [Brent]: Verify coverage >=97% via `nox -s coverage_report`. If coverage is <97% then review the current unit test coverage report at `build/coverage.xml` and use it to write new Behave based unit tests to improve code coverage. Specifically, write Behave style unit tests that are descriptively named and specifically improves coverage on whichever file has the most uncovered lines by writing tests that will target the uncovered lines in the report. Once that is done rerun `nox -s coverage_report` to verify all tests pass and coverage is above >=97%. Only mark this as complete once coverage is >=97%, if not repeat this task as many times as is needed until coverage reaches >=97%.
|
||||
- [X] Quality [Brent]: Run `nox` (all default sessions, including benchmark).
|
||||
- [X] Git [Brent]: `git add .`
|
||||
- [X] Git [Brent]: `git commit -m "test(domain): add subplan model suites"`
|
||||
- [X] Git [Brent]: `git push -u origin feature/m5-subplan-tests`
|
||||
- [X] Forgejo PR [Brent]: Open PR from `feature/m5-subplan-tests` to `master` with description "Add subplan model Behave/Robot suites and benchmarks.".
|
||||
- [X] Git [Brent]: `git checkout master`
|
||||
- [X] Git [Brent]: `git branch -d feature/m5-subplan-tests`
|
||||
|
||||
**Parallel Group E2: Subplan Spawning [Jeff + Aditya]** (depends on D2 + E1)
|
||||
- [ ] **COMMIT (Owner: Jeff | Group: E2.service | Branch: feature/m5-subplan-service | Planned: Day 26 | Expected: Day 26) - Commit message: "feat(service): add subplan service and spawn workflow"**
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Helper script for subplan model Robot Framework smoke tests.
|
||||
|
||||
Exercises SubplanConfig, SubplanStatus, SubplanFailureHandler, and Plan
|
||||
hierarchy properties without requiring the service layer or persistence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
ExecutionMode,
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
ProcessingState,
|
||||
SubplanConfig,
|
||||
SubplanFailureHandler,
|
||||
SubplanMergeStrategy,
|
||||
SubplanStatus,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Valid ULID constants
|
||||
# ---------------------------------------------------------------------------
|
||||
_ROOT_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAV"
|
||||
_PARENT_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAW"
|
||||
_CHILD_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAX"
|
||||
_GRANDCHILD_ULID = "01ARZ3NDEKTSV4RRFFQ69G5FAY"
|
||||
_SUB_STATUS_ULID = "01KH72MDGQMPRW3R5WPZVB8KC1"
|
||||
|
||||
|
||||
def _make_plan(
|
||||
*,
|
||||
plan_id: str = _ROOT_ULID,
|
||||
parent_plan_id: str | None = None,
|
||||
root_plan_id: str | None = None,
|
||||
phase: PlanPhase = PlanPhase.STRATEGIZE,
|
||||
processing_state: ProcessingState = ProcessingState.QUEUED,
|
||||
subplan_config: SubplanConfig | None = None,
|
||||
subplan_statuses: list[SubplanStatus] | None = None,
|
||||
) -> Plan:
|
||||
"""Create a Plan for testing with minimal required fields."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(
|
||||
plan_id=plan_id,
|
||||
parent_plan_id=parent_plan_id,
|
||||
root_plan_id=root_plan_id,
|
||||
),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None, namespace="local", name="robot-subplan-test"
|
||||
),
|
||||
description="Robot subplan test plan",
|
||||
action_name="local/robot-subplan-action",
|
||||
phase=phase,
|
||||
processing_state=processing_state,
|
||||
subplan_config=subplan_config,
|
||||
subplan_statuses=subplan_statuses or [],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Command handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _root_identity() -> None:
|
||||
"""Verify root plan identity flags."""
|
||||
plan = _make_plan()
|
||||
assert plan.is_root_plan, "Expected root plan"
|
||||
assert not plan.is_subplan, "Expected not subplan"
|
||||
assert plan.depth == 0, f"Expected depth 0, got {plan.depth}"
|
||||
print("root-identity-ok")
|
||||
|
||||
|
||||
def _child_identity() -> None:
|
||||
"""Verify child plan identity flags."""
|
||||
plan = _make_plan(
|
||||
plan_id=_CHILD_ULID,
|
||||
parent_plan_id=_PARENT_ULID,
|
||||
root_plan_id=_ROOT_ULID,
|
||||
)
|
||||
assert plan.is_subplan, "Expected subplan"
|
||||
assert not plan.is_root_plan, "Expected not root plan"
|
||||
assert plan.depth == -1, f"Expected depth -1, got {plan.depth}"
|
||||
print("child-identity-ok")
|
||||
|
||||
|
||||
def _hierarchy_propagation() -> None:
|
||||
"""Verify root_plan_id propagates through three levels."""
|
||||
root = _make_plan(plan_id=_ROOT_ULID, root_plan_id=_ROOT_ULID)
|
||||
parent = _make_plan(
|
||||
plan_id=_PARENT_ULID,
|
||||
parent_plan_id=_ROOT_ULID,
|
||||
root_plan_id=_ROOT_ULID,
|
||||
)
|
||||
grandchild = _make_plan(
|
||||
plan_id=_GRANDCHILD_ULID,
|
||||
parent_plan_id=_PARENT_ULID,
|
||||
root_plan_id=_ROOT_ULID,
|
||||
)
|
||||
assert root.is_root_plan
|
||||
assert parent.is_subplan
|
||||
assert grandchild.is_subplan
|
||||
assert grandchild.identity.root_plan_id == root.identity.plan_id
|
||||
print("hierarchy-propagation-ok")
|
||||
|
||||
|
||||
def _config_defaults() -> None:
|
||||
"""Verify SubplanConfig defaults."""
|
||||
cfg = SubplanConfig()
|
||||
assert cfg.execution_mode == ExecutionMode.SEQUENTIAL
|
||||
assert cfg.merge_strategy == SubplanMergeStrategy.GIT_THREE_WAY
|
||||
assert cfg.max_parallel == 5
|
||||
assert cfg.fail_fast is False
|
||||
assert cfg.retry_failed is True
|
||||
assert cfg.max_retries == 2
|
||||
assert cfg.timeout_per_subplan_seconds is None
|
||||
print("config-defaults-ok")
|
||||
|
||||
|
||||
def _config_custom() -> None:
|
||||
"""Verify SubplanConfig accepts custom settings."""
|
||||
cfg = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
merge_strategy=SubplanMergeStrategy.FAIL_ON_CONFLICT,
|
||||
max_parallel=10,
|
||||
fail_fast=True,
|
||||
retry_failed=True,
|
||||
max_retries=4,
|
||||
timeout_per_subplan_seconds=300,
|
||||
)
|
||||
assert cfg.execution_mode == ExecutionMode.PARALLEL
|
||||
assert cfg.merge_strategy == SubplanMergeStrategy.FAIL_ON_CONFLICT
|
||||
assert cfg.max_parallel == 10
|
||||
assert cfg.fail_fast is True
|
||||
assert cfg.max_retries == 4
|
||||
assert cfg.timeout_per_subplan_seconds == 300
|
||||
print("config-custom-ok")
|
||||
|
||||
|
||||
def _failure_stop_others() -> None:
|
||||
"""Verify SubplanFailureHandler stops others when fail_fast=True."""
|
||||
config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.PARALLEL,
|
||||
fail_fast=True,
|
||||
)
|
||||
status = SubplanStatus(
|
||||
subplan_id=_SUB_STATUS_ULID,
|
||||
action_name="local/sub-task",
|
||||
status=ProcessingState.ERRORED,
|
||||
error="TimeoutError: timed out",
|
||||
)
|
||||
handler = SubplanFailureHandler()
|
||||
assert handler.should_stop_others(config, status) is True
|
||||
print("failure-stop-others-ok")
|
||||
|
||||
|
||||
def _failure_retry_retriable() -> None:
|
||||
"""Verify SubplanFailureHandler retries retriable errors."""
|
||||
config = SubplanConfig(retry_failed=True, max_retries=2)
|
||||
status = SubplanStatus(
|
||||
subplan_id=_SUB_STATUS_ULID,
|
||||
action_name="local/sub-task",
|
||||
status=ProcessingState.ERRORED,
|
||||
error="TimeoutError: timed out",
|
||||
attempt_number=1,
|
||||
)
|
||||
handler = SubplanFailureHandler()
|
||||
assert handler.should_retry(config, status) is True
|
||||
print("failure-retry-retriable-ok")
|
||||
|
||||
|
||||
def _failure_skip_non_retriable() -> None:
|
||||
"""Verify SubplanFailureHandler does not retry non-retriable errors."""
|
||||
config = SubplanConfig(retry_failed=True, max_retries=2)
|
||||
status = SubplanStatus(
|
||||
subplan_id=_SUB_STATUS_ULID,
|
||||
action_name="local/sub-task",
|
||||
status=ProcessingState.ERRORED,
|
||||
error="ConfigurationError: bad config",
|
||||
attempt_number=1,
|
||||
)
|
||||
handler = SubplanFailureHandler()
|
||||
assert handler.should_retry(config, status) is False
|
||||
print("failure-skip-non-retriable-ok")
|
||||
|
||||
|
||||
def _cli_dict_child() -> None:
|
||||
"""Verify as_cli_dict includes parent_plan_id for child plan."""
|
||||
plan = _make_plan(
|
||||
plan_id=_CHILD_ULID,
|
||||
parent_plan_id=_PARENT_ULID,
|
||||
root_plan_id=_ROOT_ULID,
|
||||
)
|
||||
cli = plan.as_cli_dict()
|
||||
assert "parent_plan_id" in cli, "Expected parent_plan_id in CLI dict"
|
||||
assert cli["parent_plan_id"] == _PARENT_ULID
|
||||
print("cli-dict-child-ok")
|
||||
|
||||
|
||||
def _cli_dict_subplan_count() -> None:
|
||||
"""Verify as_cli_dict includes subplan_count when subplans exist."""
|
||||
statuses = [
|
||||
SubplanStatus(
|
||||
subplan_id=_SUB_STATUS_ULID,
|
||||
action_name="local/sub-task",
|
||||
status=ProcessingState.QUEUED,
|
||||
),
|
||||
]
|
||||
plan = _make_plan(subplan_statuses=statuses)
|
||||
cli = plan.as_cli_dict()
|
||||
assert "subplan_count" in cli, "Expected subplan_count in CLI dict"
|
||||
assert cli["subplan_count"] == 1
|
||||
print("cli-dict-subplan-count-ok")
|
||||
|
||||
|
||||
def _plan_with_config() -> None:
|
||||
"""Verify Plan model accepts subplan config and statuses."""
|
||||
config = SubplanConfig(
|
||||
execution_mode=ExecutionMode.SEQUENTIAL,
|
||||
merge_strategy=SubplanMergeStrategy.SEQUENTIAL_APPLY,
|
||||
)
|
||||
statuses = [
|
||||
SubplanStatus(
|
||||
subplan_id=_SUB_STATUS_ULID,
|
||||
action_name="local/sub-task",
|
||||
target_resources=["src/main.py"],
|
||||
status=ProcessingState.QUEUED,
|
||||
),
|
||||
]
|
||||
plan = _make_plan(subplan_config=config, subplan_statuses=statuses)
|
||||
assert plan.has_subplans
|
||||
assert plan.subplan_config is not None
|
||||
assert plan.subplan_config.execution_mode == ExecutionMode.SEQUENTIAL
|
||||
assert len(plan.subplan_statuses) == 1
|
||||
print("plan-with-config-ok")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Dispatch command from sys.argv."""
|
||||
if len(sys.argv) < 2:
|
||||
raise SystemExit("Expected command argument")
|
||||
command = sys.argv[1]
|
||||
commands: dict[str, object] = {
|
||||
"root-identity": _root_identity,
|
||||
"child-identity": _child_identity,
|
||||
"hierarchy-propagation": _hierarchy_propagation,
|
||||
"config-defaults": _config_defaults,
|
||||
"config-custom": _config_custom,
|
||||
"failure-stop-others": _failure_stop_others,
|
||||
"failure-retry-retriable": _failure_retry_retriable,
|
||||
"failure-skip-non-retriable": _failure_skip_non_retriable,
|
||||
"cli-dict-child": _cli_dict_child,
|
||||
"cli-dict-subplan-count": _cli_dict_subplan_count,
|
||||
"plan-with-config": _plan_with_config,
|
||||
}
|
||||
if command not in commands:
|
||||
raise SystemExit(f"Unknown command: {command}")
|
||||
func = commands[command]
|
||||
if callable(func):
|
||||
func()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
*** Settings ***
|
||||
Documentation Smoke tests for subplan model hierarchy, configuration defaults,
|
||||
... failure handling, and CLI status surface rendering.
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} robot/helper_subplan_model.py
|
||||
|
||||
*** Test Cases ***
|
||||
Root Plan Identity Flags
|
||||
[Documentation] Verify root plan has is_root_plan=True, is_subplan=False, depth=0
|
||||
[Tags] subplan model identity
|
||||
${result}= Run Process ${PYTHON} ${HELPER} root-identity cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} root-identity-ok
|
||||
|
||||
Child Plan Identity Flags
|
||||
[Documentation] Verify child plan has is_subplan=True, is_root_plan=False, depth=-1
|
||||
[Tags] subplan model identity
|
||||
${result}= Run Process ${PYTHON} ${HELPER} child-identity cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} child-identity-ok
|
||||
|
||||
Three Level Hierarchy Root Propagation
|
||||
[Documentation] Verify root_plan_id propagates through three levels
|
||||
[Tags] subplan model hierarchy
|
||||
${result}= Run Process ${PYTHON} ${HELPER} hierarchy-propagation cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} hierarchy-propagation-ok
|
||||
|
||||
SubplanConfig Defaults
|
||||
[Documentation] Verify SubplanConfig defaults are sensible
|
||||
[Tags] subplan model config
|
||||
${result}= Run Process ${PYTHON} ${HELPER} config-defaults cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} config-defaults-ok
|
||||
|
||||
SubplanConfig Custom Settings
|
||||
[Documentation] Verify SubplanConfig accepts custom parallel/retry settings
|
||||
[Tags] subplan model config
|
||||
${result}= Run Process ${PYTHON} ${HELPER} config-custom cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} config-custom-ok
|
||||
|
||||
Failure Handler Stop Others On Fail Fast
|
||||
[Documentation] Verify SubplanFailureHandler stops others when fail_fast=True
|
||||
[Tags] subplan model failure
|
||||
${result}= Run Process ${PYTHON} ${HELPER} failure-stop-others cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} failure-stop-others-ok
|
||||
|
||||
Failure Handler Retry Retriable Error
|
||||
[Documentation] Verify SubplanFailureHandler retries retriable TimeoutError
|
||||
[Tags] subplan model failure retry
|
||||
${result}= Run Process ${PYTHON} ${HELPER} failure-retry-retriable cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} failure-retry-retriable-ok
|
||||
|
||||
Failure Handler Skip Non Retriable Error
|
||||
[Documentation] Verify SubplanFailureHandler skips non-retriable ConfigurationError
|
||||
[Tags] subplan model failure retry
|
||||
${result}= Run Process ${PYTHON} ${HELPER} failure-skip-non-retriable cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} failure-skip-non-retriable-ok
|
||||
|
||||
CLI Dict Renders Parent Plan Id For Child
|
||||
[Documentation] Verify as_cli_dict includes parent_plan_id for child plans
|
||||
[Tags] subplan model cli
|
||||
${result}= Run Process ${PYTHON} ${HELPER} cli-dict-child cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-dict-child-ok
|
||||
|
||||
CLI Dict Renders Subplan Count
|
||||
[Documentation] Verify as_cli_dict includes subplan_count when subplans exist
|
||||
[Tags] subplan model cli
|
||||
${result}= Run Process ${PYTHON} ${HELPER} cli-dict-subplan-count cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} cli-dict-subplan-count-ok
|
||||
|
||||
Plan With Subplan Config And Statuses
|
||||
[Documentation] Verify Plan model accepts subplan config and status list
|
||||
[Tags] subplan model plan
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-with-config cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} plan-with-config-ok
|
||||
Reference in New Issue
Block a user