test(integration): workflow example 12 — large-scale hierarchical feature implementation (supervised profile) #952

Closed
brent.edwards wants to merge 1 commits from test/int-wf12-hierarchical into master
3 changed files with 567 additions and 0 deletions
+8
View File
@@ -2,6 +2,14 @@
## Unreleased
- Added Robot Framework integration test and Python helper for Specification
Workflow Example 12 (large-scale hierarchical feature implementation).
Covers multi-project setup (4 projects), hierarchical child plan
decomposition with 4+ levels, subplan creation, `plan correct --mode
append`, phased apply in dependency order, and decision tree navigation
— all using mocked LLM providers under the supervised automation
profile. (`robot/wf12_hierarchical.robot`,
`robot/helper_wf12_hierarchical.py`) (#776)
- Added TDD bug-capture tests for bug #1076`use_action()` does not
propagate `automation_profile` to Plan. Three Behave BDD scenarios
(`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence
+481
View File
@@ -0,0 +1,481 @@
"""Robot helper for Workflow Example 12 — hierarchical feature implementation.
Exercises multi-project setup (4 projects), hierarchical child plan
decomposition, plan tree, plan correct --mode append, and phased apply.
"""
# ruff: noqa: E402
from __future__ import annotations
import json
import os
import re
import sys
from collections.abc import Callable
from pathlib import Path
from typing import Any, NoReturn
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
_ROBOT = str(Path(__file__).resolve().parent)
if _ROBOT not in sys.path:
sys.path.insert(0, _ROBOT)
from helper_e2e_common import cleanup_workspace, run_cli, setup_workspace, write_yaml
from helpers_common import reset_global_state
from cleveragents.application.services.decision_service import DecisionService
from cleveragents.config.settings import Settings
from cleveragents.domain.models.core.decision import (
ContextSnapshot,
Decision,
DecisionType,
ResourceRef,
)
from cleveragents.infrastructure.database.unit_of_work import UnitOfWork
_WIDE: dict[str, str] = {"COLUMNS": "500"}
_RES_API = "01HXM8A1ZK4Q7C2B3F2R4VYV6A"
_RES_PROTOS = "01HXM8D4ZK4Q7C2B3F2R4VYV6D"
_ACTION_YAML = """\
name: local/build-notification-system
description: Build a complete notification system spanning multiple services
definition_of_done: Protos complete, API implemented, workers deployed, frontend live.
strategy_actor: openai/gpt-4
execution_actor: openai/gpt-4
automation_profile: cautious
arguments:
- name: notification_channels
type: string
required: true
description: Comma-separated delivery channels
- name: priority_levels
type: string
required: false
default: "critical,high,normal,low"
- name: max_retry_attempts
type: integer
required: false
default: 5
invariants:
- "Proto definitions must be implemented before any service code"
- "Each service must be deployable independently"
- "All notification delivery must be async"
"""
_PROJ_API = "local/api"
_PROJ_WORKER = "local/worker"
_PROJ_FRONTEND = "local/frontend"
_PROJ_PROTOS = "local/protos"
def _fail(msg: str) -> NoReturn:
print(f"FAIL: {msg}", file=sys.stderr)
raise SystemExit(1)
def _extract_plan_id(output: str) -> str | None:
m = re.search(r"\b([0-9A-Z]{26})\b", output)
return m.group(1) if m else None
def _rejoin(text: str) -> str:
"""Rejoin lines broken by Rich inside JSON string values."""
lines: list[str] = text.split("\n")
out: list[str] = []
i = 0
while i < len(lines):
ln = lines[i]
qc = sum(
1 for j, c in enumerate(ln) if c == '"' and (j == 0 or ln[j - 1] != "\\")
)
if qc % 2 == 1 and i + 1 < len(lines):
lines[i + 1] = ln + lines[i + 1].lstrip()
i += 1
continue
out.append(ln)
i += 1
return "\n".join(out)
def _load_json(output: str) -> Any:
"""Parse CLI *output* as JSON, tolerating Rich line-wrapping."""
text = output.strip()
for cand in (text, _rejoin(text)):
try:
return json.loads(cand)
except json.JSONDecodeError:
pass
dec = json.JSONDecoder()
for cand in (text, _rejoin(text)):
for i, ch in enumerate(cand):
if ch not in "[{":
continue
try:
val, end = dec.raw_decode(cand[i:])
except json.JSONDecodeError:
continue
if not cand[i:][end:].strip():
return val
_fail(f"invalid JSON:\n{output[:500]}")
def _no_crash(combined: str, label: str) -> None:
if "INTERNAL" in combined or "Traceback" in combined:
_fail(f"{label} crashed:\n{combined}")
def _cli(*args: str, ws: str) -> Any:
r = run_cli(*args, workspace=ws, env_extra=_WIDE)
_no_crash(r.stdout + r.stderr, args[0] if args else "cli")
return r
def _settings(db_url: str) -> Settings:
prev = os.environ.get("CLEVERAGENTS_DATABASE_URL")
os.environ["CLEVERAGENTS_DATABASE_URL"] = db_url
try:
return Settings()
finally:
if prev is None:
os.environ.pop("CLEVERAGENTS_DATABASE_URL", None)
else:
os.environ["CLEVERAGENTS_DATABASE_URL"] = prev
def _snap(prefix: str, res_id: str, path: str) -> ContextSnapshot:
return ContextSnapshot(
hot_context_hash=f"sha256:wf12_{prefix}",
hot_context_ref=f"store://snapshots/wf12_{prefix}",
relevant_resources=[ResourceRef(resource_id=res_id, path=path)],
actor_state_ref=f"checkpoint://actor/wf12_{prefix}",
)
def _get_dsvc(db_url: str) -> DecisionService:
return DecisionService(
settings=_settings(db_url),
unit_of_work=UnitOfWork(db_url),
)
def _create_action_and_plan(
ws: str,
*,
projects: list[str] | None = None,
args: list[str] | None = None,
) -> str:
"""Create action + plan, return plan_id."""
yp = write_yaml(_ACTION_YAML)
try:
r = run_cli("action", "create", "--config", yp, workspace=ws, env_extra=_WIDE)
if r.returncode != 0:
_fail(f"action create: {r.stderr}")
finally:
os.unlink(yp)
plan_args: list[str] = [
"plan",
"use",
"local/build-notification-system",
]
for p in projects or [_PROJ_API]:
plan_args.append(p)
for a in args or []:
plan_args.extend(["--arg", a])
plan_args.extend(["--format", "json"])
r2 = run_cli(*plan_args, workspace=ws, env_extra=_WIDE)
if r2.returncode != 0:
_fail(f"plan use: {r2.stderr}")
pd = _load_json(r2.stdout)
pid = pd.get("plan_id") if isinstance(pd, dict) else _extract_plan_id(r2.stdout)
if not isinstance(pid, str) or not pid:
_fail(f"missing plan_id in plan use output: {pd}")
return pid
def _seed_hierarchical_decisions(
svc: DecisionService,
plan_id: str,
) -> tuple[Decision, Decision, Decision, Decision]:
"""Seed a parent plan's decision tree with child-plan structure.
Returns (root, strategy, subplan_protos, subplan_api).
"""
root = svc.record_decision(
plan_id=plan_id,
decision_type=DecisionType.PROMPT_DEFINITION,
question="Build notification system across 4 projects",
chosen_option="Decompose into phased child plans per project",
alternatives_considered=["Single monolithic plan", "Two parallel plans"],
confidence_score=0.92,
rationale="Phased decomposition respects project dependencies.",
context_snapshot=_snap("root", _RES_API, "src/notifications/init.py"),
)
strategy = svc.record_decision(
plan_id=plan_id,
decision_type=DecisionType.STRATEGY_CHOICE,
question="Phase 1: proto definitions first (blocking)?",
chosen_option="Yes — protos block all downstream services",
parent_decision_id=root.decision_id,
alternatives_considered=["Start API and protos in parallel"],
confidence_score=0.95,
rationale="Proto definitions are a shared dependency.",
context_snapshot=_snap("strategy", _RES_PROTOS, "protos/notification.proto"),
)
subplan_protos = svc.record_decision(
plan_id=plan_id,
decision_type=DecisionType.SUBPLAN_SPAWN,
question="Spawn child plan for proto definitions",
chosen_option="Phase 1 child: local/protos",
parent_decision_id=strategy.decision_id,
alternatives_considered=[],
confidence_score=0.98,
rationale="Proto definitions must be completed first.",
context_snapshot=_snap("child_protos", _RES_PROTOS, "protos/events.proto"),
)
subplan_api = svc.record_decision(
plan_id=plan_id,
decision_type=DecisionType.SUBPLAN_SPAWN,
question="Spawn child plan for API endpoints",
chosen_option="Phase 2 child: local/api",
parent_decision_id=strategy.decision_id,
alternatives_considered=[],
confidence_score=0.90,
rationale="API endpoints depend on proto definitions.",
context_snapshot=_snap("child_api", _RES_API, "src/api/notifications.py"),
)
return root, strategy, subplan_protos, subplan_api
def wf12_multi_project_setup() -> None:
"""Register 4 projects with invariants and a global invariant."""
ws = setup_workspace(prefix="wf12_proj_")
try:
r = _cli(
"invariant",
"add",
"--global",
"All inter-service communication must use shared proto definitions",
"--format",
"json",
ws=ws,
)
if r.returncode != 0:
_fail(f"invariant add --global rc={r.returncode}\n{r.stderr}")
d = _load_json(r.stdout)
if not isinstance(d, dict) or d.get("scope") != "global":
_fail(f"global invariant scope mismatch: {d}")
projects: list[tuple[str, str, list[str]]] = [
(_PROJ_API, "Backend API", ["All new endpoints must have OpenAPI docs"]),
(_PROJ_WORKER, "Background worker service", ["Workers must be idempotent"]),
(
_PROJ_FRONTEND,
"Frontend dashboard",
["All components must have Storybook stories"],
),
(
_PROJ_PROTOS,
"Shared protobuf definitions",
["Proto changes must be backward-compatible"],
),
]
for name, desc, invariants in projects:
cmd: list[str] = ["project", "create", "-d", desc]
for inv in invariants:
cmd.extend(["--invariant", inv])
cmd.extend(["--format", "json", name])
r = _cli(*cmd, ws=ws)
if r.returncode != 0:
_fail(f"project create {name}: {r.stderr}")
_no_crash(r.stdout + r.stderr, f"project create {name}")
print("wf12-projects-ok")
finally:
cleanup_workspace(ws)
def wf12_hierarchical_plan_creation() -> None:
"""Create action with cautious profile, plan use with 4 projects."""
ws = setup_workspace(prefix="wf12_plan_")
try:
pid = _create_action_and_plan(
ws,
projects=[_PROJ_PROTOS, _PROJ_API, _PROJ_WORKER, _PROJ_FRONTEND],
args=[
"notification_channels=email,push,in-app",
"priority_levels=critical,high,normal,low",
"max_retry_attempts=5",
],
)
r = _cli("plan", "status", pid, "--format", "json", ws=ws)
if r.returncode != 0:
_fail(f"plan status rc={r.returncode}\n{r.stderr}")
sd = _load_json(r.stdout)
if not isinstance(sd, dict):
_fail(f"status not dict: {sd}")
if sd.get("plan_id") != pid:
_fail(f"plan_id mismatch: {sd.get('plan_id')} != {pid}")
if not sd.get("phase"):
_fail(f"missing phase: {sd}")
print("wf12-plan-ok")
finally:
cleanup_workspace(ws)
def wf12_tree_with_child_plans() -> None:
"""Seed parent+child plan decisions, verify plan tree structure."""
ws = setup_workspace(prefix="wf12_tree_")
try:
pid = _create_action_and_plan(
ws,
projects=[_PROJ_API],
args=["notification_channels=email,push,in-app"],
)
db_url = os.environ["CLEVERAGENTS_DATABASE_URL"]
dsvc = _get_dsvc(db_url)
root, strategy, sp_protos, sp_api = _seed_hierarchical_decisions(dsvc, pid)
r = _cli("plan", "tree", pid, "--format", "json", ws=ws)
if r.returncode != 0:
_fail(f"plan tree rc={r.returncode}\n{r.stderr}")
tree = _load_json(r.stdout)
if not isinstance(tree, list) or len(tree) != 1:
_fail(f"expected 1 root node, got: {tree}")
rn = tree[0]
if rn.get("decision_id") != root.decision_id:
_fail(f"root mismatch: {rn.get('decision_id')}")
cn = rn.get("children")
if not isinstance(cn, list) or len(cn) != 1:
_fail(f"expected 1 strategy child: {cn}")
if cn[0].get("decision_id") != strategy.decision_id:
_fail(f"strategy mismatch: {cn[0].get('decision_id')}")
gn = cn[0].get("children")
if not isinstance(gn, list) or len(gn) != 2:
_fail(f"expected 2 subplan children: {gn}")
gn_ids = {g.get("decision_id") for g in gn}
if sp_protos.decision_id not in gn_ids:
_fail(f"protos subplan missing: {gn_ids}")
if sp_api.decision_id not in gn_ids:
_fail(f"api subplan missing: {gn_ids}")
print("wf12-tree-ok")
finally:
cleanup_workspace(ws)
def wf12_correct_mode_append() -> None:
"""Correct a decision via plan correct --mode append."""
ws = setup_workspace(prefix="wf12_correct_")
try:
pid = _create_action_and_plan(
ws,
projects=[_PROJ_API],
args=["notification_channels=email,push,in-app"],
)
db_url = os.environ["CLEVERAGENTS_DATABASE_URL"]
dsvc = _get_dsvc(db_url)
_, _, sp_protos, _ = _seed_hierarchical_decisions(dsvc, pid)
r = _cli(
"plan",
"correct",
sp_protos.decision_id,
"--mode",
"append",
"--guidance",
"Skip SMS delivery for initial implementation. "
"Add a placeholder worker that logs a warning.",
"--plan",
pid,
"--yes",
"--format",
"json",
ws=ws,
)
_no_crash(r.stdout + r.stderr, "plan correct --mode append")
if r.returncode == 0:
cd = _load_json(r.stdout)
if not isinstance(cd, dict):
_fail(f"correct not dict: {cd}")
if cd.get("status") != "applied":
_fail(f"correct status mismatch: {cd}")
r2 = _cli("plan", "status", pid, "--format", "json", ws=ws)
if r2.returncode != 0:
_fail(f"plan status after correct rc={r2.returncode}\n{r2.stderr}")
sd = _load_json(r2.stdout)
if not isinstance(sd, dict) or sd.get("plan_id") != pid:
_fail(f"status mismatch after correct: {sd}")
print("wf12-correct-ok")
finally:
cleanup_workspace(ws)
def wf12_phased_apply() -> None:
"""Test plan lifecycle-apply, verify response envelope."""
ws = setup_workspace(prefix="wf12_apply_")
try:
pid = _create_action_and_plan(
ws,
projects=[_PROJ_API],
args=["notification_channels=email,push,in-app"],
)
# lifecycle-apply on a strategize-phase plan will fail with a
# clear error; we verify the CLI surfaces this without crashing.
r = _cli("plan", "lifecycle-apply", pid, "--format", "json", ws=ws)
_no_crash(r.stdout + r.stderr, "plan lifecycle-apply")
combined = r.stdout + r.stderr
if not combined.strip():
_fail("lifecycle-apply produced no output")
r2 = _cli("plan", "status", pid, "--format", "json", ws=ws)
if r2.returncode != 0:
_fail(f"plan status after apply rc={r2.returncode}\n{r2.stderr}")
sd = _load_json(r2.stdout)
if not isinstance(sd, dict) or sd.get("plan_id") != pid:
_fail(f"status mismatch after apply: {sd}")
print("wf12-apply-ok")
finally:
cleanup_workspace(ws)
_COMMANDS: dict[str, Callable[[], None]] = {
"multi-project-setup": wf12_multi_project_setup,
"hierarchical-plan-creation": wf12_hierarchical_plan_creation,
"tree-with-child-plans": wf12_tree_with_child_plans,
"correct-mode-append": wf12_correct_mode_append,
"phased-apply": wf12_phased_apply,
}
def main() -> int:
"""Entry point called by Robot Framework ``Run Process``."""
reset_global_state()
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
print(
f"Usage: helper_wf12_hierarchical.py <{'|'.join(_COMMANDS)}>",
file=sys.stderr,
)
return 1
_COMMANDS[sys.argv[1]]()
return 0
if __name__ == "__main__":
sys.exit(main())
+78
View File
@@ -0,0 +1,78 @@
*** Settings ***
Documentation Integration test for Specification Workflow Example 12:
... Large-Scale Feature Implementation with Hierarchical
... Decomposition.
...
... Exercises multi-project setup (4 projects), hierarchical
... child plan decomposition, plan tree with parent-child
... relationships, error handling via ``plan correct --mode
... append``, and phased apply in dependency order, all
... using mocked LLM providers.
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment
Suite Teardown Cleanup Test Environment
Force Tags wf12 integration hierarchical v3.5.0 tdd_issue tdd_issue_776
*** Variables ***
${HELPER} ${CURDIR}/helper_wf12_hierarchical.py
*** Test Cases ***
WF12 Multi Project Setup With Invariants
[Documentation] Register 4 projects (api, worker, frontend, protos)
... with per-project invariants and a global invariant.
... Verifies project creation and invariant scoping.
[Tags] project invariant multi_project
${result}= Run Process ${PYTHON} ${HELPER} multi-project-setup cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf12-projects-ok
WF12 Hierarchical Plan Creation Across Four Projects
[Documentation] Create an action with cautious profile and typed
... arguments, then ``plan use`` across 4 projects.
... Verifies the plan is created in strategize phase
... with correct plan_id and project links.
[Tags] plan cautious multi_project hierarchical
${result}= Run Process ${PYTHON} ${HELPER} hierarchical-plan-creation cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf12-plan-ok
WF12 Plan Tree With Child Plan Decisions
[Documentation] Seed hierarchical decisions (root, strategy,
... subplan_spawn nodes) and call ``plan tree
... --format json``. Verifies the tree contains
... the expected parent-child structure with
... subplan spawn decisions.
[Tags] plan_tree child_plan core_assertion
${result}= Run Process ${PYTHON} ${HELPER} tree-with-child-plans cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf12-tree-ok
WF12 Correct Mode Append For Failed Child Plan
[Documentation] Call ``plan correct --mode append --guidance "..."``
... on a subplan decision to skip SMS delivery with a
... placeholder. Verifies the correction is applied and
... the plan remains accessible via ``plan status``.
[Tags] correction append error_handling
${result}= Run Process ${PYTHON} ${HELPER} correct-mode-append cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf12-correct-ok
WF12 Phased Apply In Dependency Order
[Documentation] Call ``plan lifecycle-apply`` and verify the CLI
... responds appropriately. Confirms the apply surface
... handles plans in various phases and plan status
... remains accessible after apply attempts.
[Tags] plan_apply phased dependency_order
${result}= Run Process ${PYTHON} ${HELPER} phased-apply cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf12-apply-ok