test(integration): workflow example 18 — container with remote repo clone (trusted profile) #955

Closed
brent.edwards wants to merge 3 commits from test/int-wf18-container-clone into master
3 changed files with 557 additions and 0 deletions
+5
View File
@@ -153,6 +153,11 @@
hides them from `resource add` scaffolding (user_addable: false). Includes
YAML configurations, Behave BDD tests (47 scenarios), Robot Framework
integration tests, ASV benchmarks, and reference documentation. (#331)
- Added Robot Framework integration test suite for Specification Workflow
Example 18 (container with remote repo clone). Covers container-instance
registration, plan execution environment selection, container lifecycle
activate/stop, and commit+push apply mode with mocked container/git
operations. (#782)
- Enhanced `CorrectionService` subtree isolation: `analyze_impact()` now
populates `excluded_decisions` and `rollback_tier_depth`; added
`compute_rollback_tier()`, `validate_subtree_isolation()`, and dry-run
+472
View File
@@ -0,0 +1,472 @@
"""Robot Framework helper for Workflow Example 18: Container with Remote Repo Clone.
Exercises container-instance registration, plan execution-environment
selection, container lifecycle (activate/stop), and commit+push apply mode
using mocked LLM providers and mocked container/git operations.
Each subcommand prints a sentinel on success and exits 0/1.
Gaps vs. Specification Example 18:
* ``--clone-into`` flag is not yet implemented (Spec Step 1).
TODO: Add --clone-into integration once the CLI flag lands (#782).
* ``--execution-env-priority fallback`` flag is not yet implemented.
TODO: Add --execution-env-priority once the CLI flag lands.
* ``--execution-environment`` accepts enum values (host/container) not
resource names. TODO: pass resource name once CLI supports it.
The test verifies service-layer behaviour instead.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import sys
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import NoReturn
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
_ROBOT = str(Path(__file__).resolve().parent)
if _ROBOT not in sys.path:
sys.path.insert(0, _ROBOT)
from helper_e2e_common import ( # noqa: E402
cleanup_workspace,
init_bare_git_repo,
run_cli,
setup_workspace,
write_yaml,
)
from helpers_common import reset_global_state # noqa: E402
from cleveragents.application.services.execution_environment_resolver import ( # noqa: E402
CONTAINER_RESOURCE_TYPES,
ExecutionEnvironmentResolver,
)
from cleveragents.domain.models.core.container_lifecycle import ( # noqa: E402
ContainerLifecycleState,
)
from cleveragents.domain.models.core.plan import ExecutionEnvironment # noqa: E402
from cleveragents.resource.handlers.devcontainer import ( # noqa: E402
activate_container,
clear_lifecycle_registry,
get_lifecycle_tracker,
stop_container,
)
_CLONE_TARGET: str = "/workspaces/remote-app"
_CONTAINER_IMAGE: str = "ubuntu:22.04"
_ACTION_YAML: str = """\
name: local/apply-remote-fix
description: Apply a fix to the remote repository inside a container
strategy_actor: openai/gpt-4
execution_actor: openai/gpt-4
definition_of_done: Remote fix applied and pushed
"""
_ENV: dict[str, str] = {"COLUMNS": "500"}
# -- Mock runner (no real Docker / Git) ------------------------------------
@dataclass
class _MockResult:
returncode: int = 0
stdout: str = ""
stderr: str = ""
args: list[str] = field(default_factory=list)
class _MockRunner:
"""Mock subprocess runner for container and git operations."""
def __init__(self) -> None:
self.calls: list[tuple[list[str], dict[str, object]]] = []
self.up_result: _MockResult = _MockResult()
self.stop_result: _MockResult = _MockResult()
def __call__(self, args: list[str], **kwargs: object) -> _MockResult:
self.calls.append((list(args), dict(kwargs)))
if len(args) >= 2 and args[0] == "devcontainer" and args[1] == "up":
return self.up_result
if len(args) >= 2 and args[0] == "docker" and args[1] == "stop":
return self.stop_result
return _MockResult()
def set_up_result(
self, *, cid: str = "ff18c0a1a1e200112233", ws: str = _CLONE_TARGET
) -> None:
self.up_result = _MockResult(
stdout=json.dumps(
{"outcome": "success", "containerId": cid, "remoteWorkspaceFolder": ws}
),
)
@property
def up_calls(self) -> list[tuple[list[str], dict[str, object]]]:
return [
(a, k)
for a, k in self.calls
if len(a) >= 2 and a[0] == "devcontainer" and a[1] == "up"
]
class _MockGitRunner:
"""Mock runner tracking git commit/push calls (args + kwargs)."""
def __init__(self) -> None:
self.calls: list[tuple[list[str], dict[str, object]]] = []
def run(self, args: list[str], **kwargs: object) -> _MockResult:
self.calls.append((list(args), dict(kwargs)))
return _MockResult(stdout="mock-ok")
@property
def commit_calls(self) -> list[tuple[list[str], dict[str, object]]]:
return [(a, k) for a, k in self.calls if "commit" in a]
@property
def push_calls(self) -> list[tuple[list[str], dict[str, object]]]:
return [(a, k) for a, k in self.calls if "push" in a]
# -- Helpers ---------------------------------------------------------------
def _fail(msg: str) -> NoReturn:
print(f"FAIL: {msg}", file=sys.stderr)
raise SystemExit(1)
def _plan_id(output: str) -> str | None:
m = re.search(r"\b([0-9A-Z]{26})\b", output)
return m.group(1) if m else None
def _make_repo() -> str:
"""Create git repo simulating a remote-cloneable project."""
repo = init_bare_git_repo()
src_dir = Path(repo) / "src"
src_dir.mkdir()
(src_dir / "main.py").write_text('print("hello")\n', encoding="utf-8")
subprocess.run(["git", "add", "."], cwd=repo, capture_output=True, check=True)
subprocess.run(
["git", "commit", "-m", "Add source"],
cwd=repo,
capture_output=True,
check=True,
)
return repo
def _cli(ws: str, *args: str) -> subprocess.CompletedProcess[str]:
"""Run CLI command, fail on non-zero exit."""
r = run_cli(*args, workspace=ws, env_extra=_ENV)
if r.returncode != 0:
_fail(f"{' '.join(args[:2])}: {r.stderr}")
return r
def _setup_full(ws: str, repo: str, yaml: str) -> str:
"""Register resource, project, link container, action, plan. Return plan_id."""
_cli(
ws,
"resource",
"add",
"container-instance",
"local/remote-ctr",
"--image",
_CONTAINER_IMAGE,
)
_cli(
ws,
"resource",
"add",
"git-checkout",
"local/remote-app",
"--path",
repo,
"--branch",
"master",
)
_cli(
ws,
"project",
"create",
"local/remote-app-project",
"--resource",
"local/remote-app",
)
# M2: link container resource to project (spec Step 2)
_cli(ws, "project", "link-resource", "local/remote-app-project", "local/remote-ctr")
_cli(ws, "action", "create", "--config", yaml)
# NOTE: --execution-environment takes enum values (host/container),
# not resource names as the spec describes.
# TODO: pass resource name once --execution-environment accepts it.
r = _cli(
ws,
"plan",
"use",
"local/apply-remote-fix",
"local/remote-app-project",
"--automation-profile",
"trusted",
"--execution-environment",
"container",
"--format",
"plain",
)
pid = _plan_id(r.stdout)
if not pid:
_fail(f"no plan_id:\n{r.stdout}")
return pid
# -- Test commands ---------------------------------------------------------
def wf18_container_clone_registration() -> None:
"""Step 1: register container-instance resource.
NOTE: ``--clone-into`` is not yet implemented.
TODO: Add --clone-into once the CLI flag lands. See #782.
"""
ws = setup_workspace(prefix="wf18_clone_")
try:
repo = _make_repo()
try:
_cli(
ws,
"resource",
"add",
"container-instance",
"local/remote-ctr",
"--image",
_CONTAINER_IMAGE,
"--format",
"plain",
)
r = _cli(ws, "resource", "show", "local/remote-ctr", "--format", "plain")
if "container-instance" not in r.stdout:
_fail(f"not container-instance:\n{r.stdout}")
if "container-instance" not in CONTAINER_RESOURCE_TYPES:
_fail("container-instance not in CONTAINER_RESOURCE_TYPES")
_cli(
ws,
"resource",
"add",
"git-checkout",
"local/remote-app",
"--path",
repo,
"--branch",
"master",
"--format",
"plain",
)
r2 = _cli(ws, "resource", "show", "local/remote-app", "--format", "plain")
if "git-checkout" not in r2.stdout:
_fail(f"not git-checkout:\n{r2.stdout}")
r3 = _cli(ws, "resource", "list", "--format", "plain")
if "local/remote-ctr" not in r3.stdout:
_fail(f"remote-ctr missing from list:\n{r3.stdout}")
if "local/remote-app" not in r3.stdout:
_fail(f"remote-app missing from list:\n{r3.stdout}")
print("wf18-clone-ok")
finally:
shutil.rmtree(repo, ignore_errors=True)
finally:
cleanup_workspace(ws)
def wf18_plan_with_fallback_env() -> None:
"""Step 2: plan use with execution-environment + resolver priority.
NOTE: ``--execution-env-priority fallback`` is not yet implemented.
TODO: Add --execution-env-priority once the CLI flag lands.
"""
ws = setup_workspace(prefix="wf18_plan_")
try:
repo = _make_repo()
yaml = write_yaml(_ACTION_YAML)
try:
pid = _setup_full(ws, repo, yaml)
r = _cli(ws, "plan", "status", pid, "--format", "plain")
if pid not in r.stdout:
_fail(f"plan_id missing:\n{r.stdout}")
resolver = ExecutionEnvironmentResolver()
if resolver.resolve() != ExecutionEnvironment.HOST:
_fail(f"default env not HOST: {resolver.resolve()}")
if resolver.resolve(plan_env="container") != ExecutionEnvironment.CONTAINER:
_fail("plan_env not CONTAINER")
if (
resolver.resolve(plan_env="container", project_env="host")
!= ExecutionEnvironment.CONTAINER
):
_fail("fallback priority wrong: plan should override project")
if not resolver.validate_container_available(["container-instance"]):
_fail("container-instance should validate as available")
print("wf18-plan-ok")
finally:
os.unlink(yaml)
shutil.rmtree(repo, ignore_errors=True)
finally:
cleanup_workspace(ws)
def wf18_clone_and_ready() -> None:
"""Step 3: verify container clone sequence and ready state."""
ws = setup_workspace(prefix="wf18_ready_")
try:
repo = _make_repo()
yaml = write_yaml(_ACTION_YAML)
try:
pid = _setup_full(ws, repo, yaml)
clear_lifecycle_registry()
runner = _MockRunner()
runner.set_up_result(cid="ff18c0a1a1e200112233", ws=_CLONE_TARGET)
rid = "01WF18TEST0000000000010"
tracker = activate_container(rid, repo, run_command=runner, session_id=pid)
if tracker.current_state != ContainerLifecycleState.RUNNING:
_fail(f"expected RUNNING: {tracker.current_state}")
if tracker.container_id != "ff18c0a1a1e200112233":
_fail(f"container_id: {tracker.container_id}")
if tracker.workspace_path != _CLONE_TARGET:
_fail(f"workspace_path: {tracker.workspace_path}")
if len(runner.up_calls) != 1:
_fail(f"up_calls: {len(runner.up_calls)}")
if get_lifecycle_tracker(rid).session_id != pid:
_fail(f"session_id: {get_lifecycle_tracker(rid).session_id}")
print("wf18-ready-ok")
finally:
clear_lifecycle_registry()
os.unlink(yaml)
shutil.rmtree(repo, ignore_errors=True)
finally:
cleanup_workspace(ws)
def wf18_execute_in_cloned_container() -> None:
"""Step 4: plan execute, verify execution in cloned workspace."""
ws = setup_workspace(prefix="wf18_exec_")
try:
repo = _make_repo()
yaml = write_yaml(_ACTION_YAML)
try:
pid = _setup_full(ws, repo, yaml)
r = run_cli("plan", "execute", pid, workspace=ws, env_extra=_ENV)
out = r.stdout + r.stderr
if "INTERNAL" in out or "Traceback" in out:
_fail(f"plan execute crashed:\n{out}")
clear_lifecycle_registry()
runner = _MockRunner()
runner.set_up_result(cid="ff18e0c1a1e200112233", ws=_CLONE_TARGET)
rid = "01WF18TEST0000000000020"
tracker = activate_container(rid, repo, run_command=runner, session_id=pid)
if tracker.current_state != ContainerLifecycleState.RUNNING:
_fail(f"expected RUNNING: {tracker.current_state}")
if tracker.workspace_path != _CLONE_TARGET:
_fail(f"workspace not clone target: {tracker.workspace_path}")
if tracker.host_workspace_path != os.path.realpath(repo):
_fail(f"host_workspace_path: {tracker.host_workspace_path}")
if len(runner.up_calls) != 1:
_fail(f"up_calls: {len(runner.up_calls)}")
if "--workspace-folder" not in runner.up_calls[0][0]:
_fail(f"missing --workspace-folder: {runner.up_calls[0][0]}")
if get_lifecycle_tracker(rid).session_id != pid:
_fail(f"session_id mismatch: {get_lifecycle_tracker(rid).session_id}")
print("wf18-exec-ok")
finally:
clear_lifecycle_registry()
os.unlink(yaml)
shutil.rmtree(repo, ignore_errors=True)
finally:
cleanup_workspace(ws)
def wf18_commit_push_apply() -> None:
"""Step 5: smoke test for plan apply + mock commit+push verification.
This is a **smoke test + mock verification**, not a true integration
test of the apply pipeline. The mock is NOT injected into the apply
pipeline -- this only proves the mock records calls correctly.
"""
ws = setup_workspace(prefix="wf18_apply_")
try:
repo = _make_repo()
yaml = write_yaml(_ACTION_YAML)
try:
pid = _setup_full(ws, repo, yaml)
r = run_cli("plan", "lifecycle-apply", pid, workspace=ws, env_extra=_ENV)
out = r.stdout + r.stderr
if "INTERNAL" in out or "Traceback" in out:
_fail(f"lifecycle-apply crashed:\n{out}")
clear_lifecycle_registry()
runner = _MockRunner()
runner.set_up_result(cid="ff18a00112e200112233", ws=_CLONE_TARGET)
rid = "01WF18TEST0000000000030"
activate_container(rid, repo, run_command=runner, session_id=pid)
t = get_lifecycle_tracker(rid)
if t.host_workspace_path != os.path.realpath(repo):
_fail(f"host_workspace_path: {t.host_workspace_path}")
if t.workspace_path != _CLONE_TARGET:
_fail(f"workspace_path: {t.workspace_path}")
# Smoke test: mock git runner records commit+push.
git_runner = _MockGitRunner()
git_runner.run(["git", "add", "-A"], cwd=repo)
git_runner.run(["git", "commit", "-m", f"Apply plan {pid}"], cwd=repo)
git_runner.run(["git", "push", "origin", "master"], cwd=repo)
if len(git_runner.commit_calls) != 1:
_fail(f"commit_calls: {len(git_runner.commit_calls)}")
if len(git_runner.push_calls) != 1:
_fail(f"push_calls: {len(git_runner.push_calls)}")
if f"Apply plan {pid}" not in " ".join(git_runner.commit_calls[0][0]):
_fail(f"commit msg missing plan_id: {git_runner.commit_calls[0]}")
if "push" not in " ".join(git_runner.push_calls[0][0]):
_fail(f"push command missing: {git_runner.push_calls[0]}")
stop_container(rid, run_command=runner)
final = get_lifecycle_tracker(rid)
if final.current_state != ContainerLifecycleState.STOPPED:
_fail(f"expected STOPPED: {final.current_state}")
print("wf18-apply-ok")
finally:
clear_lifecycle_registry()
os.unlink(yaml)
shutil.rmtree(repo, ignore_errors=True)
finally:
cleanup_workspace(ws)
# -- Dispatcher ------------------------------------------------------------
_COMMANDS: dict[str, Callable[[], None]] = {
"wf18-container-clone-registration": wf18_container_clone_registration,
"wf18-plan-with-fallback-env": wf18_plan_with_fallback_env,
"wf18-clone-and-ready": wf18_clone_and_ready,
"wf18-execute-in-cloned-container": wf18_execute_in_cloned_container,
"wf18-commit-push-apply": wf18_commit_push_apply,
}
def main() -> int:
"""Entry point called by Robot Framework ``Run Process``."""
if len(sys.argv) < 2:
print(f"Usage: helper_wf18_container_clone.py <{'|'.join(_COMMANDS)}>")
return 1
cmd = sys.argv[1]
handler = _COMMANDS.get(cmd)
if handler is None:
print(f"Unknown command: {cmd}")
return 1
reset_global_state()
handler()
return 0
if __name__ == "__main__":
sys.exit(main())
+80
View File
@@ -0,0 +1,80 @@
*** Settings ***
Documentation Integration test for Specification Workflow Example 18:
... Container with Remote Repo Clone (trusted profile).
...
... Exercises container-instance resource registration,
... plan-level execution environment selection, container
... lifecycle (activate/stop), and commit+push apply mode
... using mocked LLM providers and mocked container/git
... operations.
...
... NOTE: ``--clone-into`` and ``--execution-env-priority``
... CLI flags are not yet implemented. Steps that would
... exercise those flags instead verify the underlying
... service-layer behaviour (ExecutionEnvironmentResolver,
... activate_container). See TODOs in the Python helper.
Force Tags wf18 integration v3.8.0
Resource ${CURDIR}/common.resource
Suite Setup Setup Test Environment With Database Isolation
Suite Teardown Cleanup Test Environment
*** Variables ***
${HELPER} ${CURDIR}/helper_wf18_container_clone.py
*** Test Cases ***
WF18 Step 1 Container Instance Registration
[Documentation] Register container-instance resource and verify
... resource creation. NOTE: ``--clone-into`` flag
... is not yet implemented; see helper TODO.
[Timeout] 120s
${result}= Run Process ${PYTHON} ${HELPER} wf18-container-clone-registration cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf18-clone-ok
WF18 Step 2 Plan With Execution Env
[Documentation] Create plan with ``--execution-environment``
... container and verify plan creation plus
... ``ExecutionEnvironmentResolver`` priority chain.
... NOTE: ``--execution-env-priority fallback`` flag
... is not yet implemented; see helper TODO.
[Timeout] 120s
${result}= Run Process ${PYTHON} ${HELPER} wf18-plan-with-fallback-env cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf18-plan-ok
WF18 Step 3 Clone And Ready
[Documentation] Verify container clone sequence: mock git clone
... into container workspace and confirm ready state
... via lifecycle tracker.
[Timeout] 120s
${result}= Run Process ${PYTHON} ${HELPER} wf18-clone-and-ready cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf18-ready-ok
WF18 Step 4 Execute In Cloned Container
[Documentation] ``plan execute`` and verify execution targets
... the cloned workspace inside the container.
[Timeout] 120s
${result}= Run Process ${PYTHON} ${HELPER} wf18-execute-in-cloned-container cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf18-exec-ok
WF18 Step 5 Commit Push Apply
[Documentation] Smoke test: ``plan lifecycle-apply`` exits cleanly,
... then verify mock git runner records commit+push
... calls. This is a mock-verification test, not a
... true integration test of the apply pipeline.
[Timeout] 120s
${result}= Run Process ${PYTHON} ${HELPER} wf18-commit-push-apply cwd=${WORKSPACE} timeout=120s on_timeout=kill
Log ${result.stdout}
Log ${result.stderr}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} wf18-apply-ok