Files
cleveragents-core/features/steps/sandbox_manager_coverage_steps.py
T
freemo a808c395f9
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 17s
CI / typecheck (pull_request) Successful in 35s
CI / security (pull_request) Successful in 50s
CI / unit_tests (pull_request) Successful in 2m46s
CI / integration_tests (pull_request) Successful in 3m16s
CI / docker (pull_request) Successful in 40s
CI / coverage (pull_request) Successful in 5m6s
CI / lint (push) Successful in 13s
CI / quality (push) Successful in 16s
CI / build (push) Successful in 18s
CI / security (push) Successful in 32s
CI / typecheck (push) Successful in 35s
CI / benchmark-regression (push) Has been skipped
CI / unit_tests (push) Successful in 2m52s
CI / integration_tests (push) Successful in 3m8s
CI / docker (push) Successful in 39s
CI / coverage (push) Successful in 5m53s
CI / benchmark-publish (push) Successful in 16m55s
CI / benchmark-regression (pull_request) Successful in 33m0s
test(coverage): add Behave BDD tests to improve unit test coverage across 53 source modules
Add 53 new .feature files and corresponding step definition files targeting
uncovered lines identified in build/coverage.xml. Fix AmbiguousStep conflicts
in 7 pre-existing step files by disambiguating step text.

New tests cover: ACP clients/facade, actor CLI/config, application container,
ACMS service/strategies, async worker, automation profile CLI, autonomy
guardrail, bridge, change model, config CLI/service, context service,
cross-plan correction, database models, decision service, decomposition
clustering/service, discovery handler, langchain chat provider, langgraph
nodes, materializers, multi-project service, plan apply/CLI/lifecycle/model/
preflight/resume/service, PostgreSQL analyzer, project CLI/context CLI,
provider registry, reactive application/route, repositories, resolver handler,
resource registry service, resume model, retry patterns, sandbox protocol,
server CLI, skill CLI/service, skills registry, subplan execution/service,
system CLI, UKO loader, UoW, and YAML template engine.

Closes #645
2026-03-09 13:01:58 -04:00

507 lines
18 KiB
Python

"""Step definitions for sandbox manager BDD coverage tests."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from unittest.mock import MagicMock, PropertyMock
from behave import given, then, when
from cleveragents.infrastructure.sandbox.factory import SandboxFactory
from cleveragents.infrastructure.sandbox.manager import SandboxManager
from cleveragents.infrastructure.sandbox.protocol import (
CommitResult,
SandboxContext,
SandboxError,
SandboxStatus,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_mock_sandbox(
sandbox_id: str = "sb-mock-001",
status: SandboxStatus = SandboxStatus.CREATED,
resource_id: str = "res-mock",
original_path: str = "/tmp/mock",
plan_id: str = "plan-mock",
) -> MagicMock:
"""Create a mock sandbox satisfying the Sandbox protocol."""
sb = MagicMock()
sb.sandbox_id = sandbox_id
type(sb).status = PropertyMock(return_value=status)
sb.context = SandboxContext(
sandbox_id=sandbox_id,
sandbox_path=original_path,
original_path=original_path,
resource_id=resource_id,
plan_id=plan_id,
created_at=datetime.now(),
)
sb.create.return_value = sb.context
sb.commit.return_value = CommitResult(
sandbox_id=sandbox_id,
success=True,
timestamp=datetime.now(),
)
sb.rollback.return_value = None
sb.cleanup.return_value = None
return sb
# ---------------------------------------------------------------------------
# Background / Given steps
# ---------------------------------------------------------------------------
@given("a sandbox factory instance")
def step_given_sandbox_factory(context: Any) -> None:
context.factory = SandboxFactory()
@given("a sandbox manager with the factory")
def step_given_sandbox_manager(context: Any) -> None:
context.manager = SandboxManager(factory=context.factory, cleanup_on_exit=False)
context.error = None
context.result = None
context.sandbox_result = None
context.commit_results = None
context.abandoned_count = None
context.first_sandbox = None
context.sandbox_list = None
@given("a sandbox manager with cleanup_on_exit disabled")
def step_given_sandbox_manager_no_cleanup(context: Any) -> None:
context.manager = SandboxManager(factory=context.factory, cleanup_on_exit=False)
@given(
'a sandbox exists for plan "{plan_id}" resource "{res_id}" path "{path}" strategy "{strategy}"'
)
def step_given_sandbox_exists(
context: Any, plan_id: str, res_id: str, path: str, strategy: str
) -> None:
sandbox = context.manager.get_or_create_sandbox(
plan_id=plan_id,
resource_id=res_id,
original_path=path,
sandbox_strategy=strategy,
)
# Store first sandbox for identity comparison
key = f"{plan_id}:{res_id}"
try:
refs = context._sandbox_refs
except (KeyError, AttributeError):
refs = {}
context._sandbox_refs = refs
refs[key] = sandbox
@given("cleanup_all is patched to raise a RuntimeError")
def step_given_cleanup_all_raises(context: Any) -> None:
"""Patch cleanup_all to raise a RuntimeError to exercise the except block in atexit handler."""
def _raising_cleanup_all(plan_id: str) -> None:
raise RuntimeError("simulated cleanup_all failure")
context.manager.cleanup_all = _raising_cleanup_all
@given('the sandbox for plan "{plan_id}" resource "{res_id}" is activated')
def step_given_sandbox_activated(context: Any, plan_id: str, res_id: str) -> None:
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
# Activating a NoSandbox: call get_path to transition CREATED -> ACTIVE
sandbox.get_path("dummy_file.txt")
@given('the sandbox for plan "{plan_id}" resource "{res_id}" is committed')
def step_given_sandbox_committed(context: Any, plan_id: str, res_id: str) -> None:
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
# Ensure it's in a committable state
if sandbox.status == SandboxStatus.CREATED:
pass # CREATED can commit directly
sandbox.commit()
@given('the sandbox for plan "{plan_id}" resource "{res_id}" is cleaned up')
def step_given_sandbox_cleaned_up(context: Any, plan_id: str, res_id: str) -> None:
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
sandbox.cleanup()
@given('the sandbox for plan "{plan_id}" resource "{res_id}" is rolled back')
def step_given_sandbox_rolled_back(context: Any, plan_id: str, res_id: str) -> None:
"""Put sandbox into ROLLED_BACK state using mock injection."""
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
# NoSandbox.rollback always raises, so we need to inject a mock
# that has ROLLED_BACK status into the manager's tracking dict
mock_sb = _make_mock_sandbox(
sandbox_id=sandbox.sandbox_id,
status=SandboxStatus.ROLLED_BACK,
resource_id=res_id,
original_path="/tmp/repo",
plan_id=plan_id,
)
context.manager._active_sandboxes[plan_id][res_id] = mock_sb
key = f"{plan_id}:{res_id}"
try:
context._sandbox_refs[key] = mock_sb
except (KeyError, AttributeError):
context._sandbox_refs = {key: mock_sb}
@given('the sandbox for plan "{plan_id}" resource "{res_id}" is in errored state')
def step_given_sandbox_errored(context: Any, plan_id: str, res_id: str) -> None:
"""Put sandbox into ERRORED state by injecting a mock."""
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
mock_sb = _make_mock_sandbox(
sandbox_id=sandbox.sandbox_id,
status=SandboxStatus.ERRORED,
resource_id=res_id,
original_path="/tmp/repo",
plan_id=plan_id,
)
context.manager._active_sandboxes[plan_id][res_id] = mock_sb
@given('the sandbox for plan "{plan_id}" resource "{res_id}" will fail on commit')
def step_given_sandbox_commit_fails(context: Any, plan_id: str, res_id: str) -> None:
"""Make the sandbox's commit method raise a SandboxError."""
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
# Replace with a mock that raises on commit
mock_sb = _make_mock_sandbox(
sandbox_id=sandbox.sandbox_id,
status=SandboxStatus.ACTIVE,
resource_id=res_id,
)
mock_sb.commit.side_effect = SandboxError("commit failed in test")
context.manager._active_sandboxes[plan_id][res_id] = mock_sb
@given('the sandbox for plan "{plan_id}" resource "{res_id}" will fail on rollback')
def step_given_sandbox_rollback_fails(context: Any, plan_id: str, res_id: str) -> None:
"""Make the sandbox's rollback method raise a SandboxError."""
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
mock_sb = _make_mock_sandbox(
sandbox_id=sandbox.sandbox_id,
status=SandboxStatus.ACTIVE,
resource_id=res_id,
)
mock_sb.rollback.side_effect = SandboxError("rollback failed in test")
context.manager._active_sandboxes[plan_id][res_id] = mock_sb
@given('the sandbox for plan "{plan_id}" resource "{res_id}" will fail on cleanup')
def step_given_sandbox_cleanup_fails(context: Any, plan_id: str, res_id: str) -> None:
"""Make the sandbox's cleanup method raise a SandboxError."""
sandbox = context.manager.get_sandbox(plan_id, res_id)
if sandbox is not None:
# Preserve current status
current_status = sandbox.status
mock_sb = _make_mock_sandbox(
sandbox_id=sandbox.sandbox_id
if hasattr(sandbox, "_sandbox_id")
else "sb-mock",
status=current_status,
resource_id=res_id,
)
mock_sb.cleanup.side_effect = SandboxError("cleanup failed in test")
context.manager._active_sandboxes[plan_id][res_id] = mock_sb
# ---------------------------------------------------------------------------
# When steps
# ---------------------------------------------------------------------------
@when("I create a sandbox manager with None factory")
def step_when_create_manager_none_factory(context: Any) -> None:
try:
context.manager = SandboxManager(factory=None, cleanup_on_exit=False)
except ValueError as exc:
context.error = exc
@when(
'I get or create a sandbox for plan "{plan_id}" resource "{res_id}" path "{path}" strategy "{strategy}"'
)
def step_when_get_or_create_sandbox(
context: Any, plan_id: str, res_id: str, path: str, strategy: str
) -> None:
try:
context.sandbox_result = context.manager.get_or_create_sandbox(
plan_id=plan_id,
resource_id=res_id,
original_path=path,
sandbox_strategy=strategy,
)
except (ValueError, SandboxError) as exc:
context.error = exc
@when("I get or create a sandbox with empty plan_id")
def step_when_get_or_create_empty_plan(context: Any) -> None:
try:
context.sandbox_result = context.manager.get_or_create_sandbox(
plan_id="",
resource_id="res-001",
original_path="/tmp/repo",
sandbox_strategy="none",
)
except (ValueError, SandboxError) as exc:
context.error = exc
@when("I get or create a sandbox with empty resource_id")
def step_when_get_or_create_empty_resource(context: Any) -> None:
try:
context.sandbox_result = context.manager.get_or_create_sandbox(
plan_id="plan-001",
resource_id="",
original_path="/tmp/repo",
sandbox_strategy="none",
)
except (ValueError, SandboxError) as exc:
context.error = exc
@when("I get or create a sandbox with empty original_path")
def step_when_get_or_create_empty_path(context: Any) -> None:
try:
context.sandbox_result = context.manager.get_or_create_sandbox(
plan_id="plan-001",
resource_id="res-001",
original_path="",
sandbox_strategy="none",
)
except (ValueError, SandboxError) as exc:
context.error = exc
@when("I commit all sandboxes with empty plan_id")
def step_when_commit_all_empty_plan(context: Any) -> None:
try:
context.commit_results = context.manager.commit_all("")
except ValueError as exc:
context.error = exc
@when("I rollback all sandboxes with empty plan_id")
def step_when_rollback_all_empty_plan(context: Any) -> None:
try:
context.manager.rollback_all("")
except ValueError as exc:
context.error = exc
@when("I cleanup all sandboxes with empty plan_id")
def step_when_cleanup_all_empty_plan(context: Any) -> None:
try:
context.manager.cleanup_all("")
except ValueError as exc:
context.error = exc
@when('I get sandbox for plan "{plan_id}" resource "{res_id}"')
def step_when_get_sandbox(context: Any, plan_id: str, res_id: str) -> None:
context.sandbox_result = context.manager.get_sandbox(plan_id, res_id)
@when('I list sandboxes for plan "{plan_id}"')
def step_when_list_sandboxes(context: Any, plan_id: str) -> None:
context.sandbox_list = context.manager.list_sandboxes(plan_id)
@when('I commit all sandboxes for plan "{plan_id}"')
def step_when_commit_all(context: Any, plan_id: str) -> None:
try:
context.commit_results = context.manager.commit_all(plan_id)
except ValueError as exc:
context.error = exc
@when('I rollback all sandboxes for plan "{plan_id}"')
def step_when_rollback_all(context: Any, plan_id: str) -> None:
try:
context.manager.rollback_all(plan_id)
except ValueError as exc:
context.error = exc
@when('I cleanup all sandboxes for plan "{plan_id}"')
def step_when_cleanup_all(context: Any, plan_id: str) -> None:
try:
context.manager.cleanup_all(plan_id)
except ValueError as exc:
context.error = exc
@when("I cleanup abandoned sandboxes")
def step_when_cleanup_abandoned(context: Any) -> None:
context.abandoned_count = context.manager.cleanup_abandoned()
@when("the atexit cleanup handler runs")
def step_when_atexit_handler(context: Any) -> None:
try:
context.manager._cleanup_on_exit_handler()
context.error = None
except Exception as exc:
context.error = exc
# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------
@then("the manager should have no active sandboxes")
def step_then_no_active_sandboxes(context: Any) -> None:
assert len(context.manager._active_sandboxes) == 0, (
f"Expected no active sandboxes, got {len(context.manager._active_sandboxes)}"
)
@then("the manager cleanup_on_exit flag should be true")
def step_then_cleanup_on_exit_true(context: Any) -> None:
manager = SandboxManager(factory=context.factory, cleanup_on_exit=True)
assert manager._cleanup_on_exit is True
@then("the manager cleanup_on_exit flag should be false")
def step_then_cleanup_on_exit_false(context: Any) -> None:
assert context.manager._cleanup_on_exit is False
@then('a sandbox ValueError should be raised with message "{message}"')
def step_then_sandbox_value_error(context: Any, message: str) -> None:
assert context.error is not None, "Expected a ValueError but no error was raised"
assert isinstance(context.error, ValueError), (
f"Expected ValueError, got {type(context.error).__name__}: {context.error}"
)
assert message in str(context.error), (
f"Expected message '{message}' in '{context.error}'"
)
@then("a sandbox should be returned")
def step_then_sandbox_returned(context: Any) -> None:
assert context.sandbox_result is not None, "Expected a sandbox but got None"
@then('the sandbox should have status "{status}"')
def step_then_sandbox_status(context: Any, status: str) -> None:
assert context.sandbox_result.status.value == status, (
f"Expected status '{status}', got '{context.sandbox_result.status.value}'"
)
@then('the sandbox should be tracked for plan "{plan_id}"')
def step_then_sandbox_tracked(context: Any, plan_id: str) -> None:
assert plan_id in context.manager._active_sandboxes, (
f"Plan '{plan_id}' not found in active sandboxes"
)
@then("the same sandbox instance should be returned")
def step_then_same_sandbox(context: Any) -> None:
# Compare with the first stored reference
for key, ref_sandbox in context._sandbox_refs.items():
plan_id, res_id = key.split(":")
current = context.manager.get_sandbox(plan_id, res_id)
if current is context.sandbox_result:
assert context.sandbox_result is ref_sandbox, (
"Expected the same sandbox instance but got a different one"
)
return
# If we got here with a result, check directly
assert context.sandbox_result is not None
@then("a different sandbox instance should be returned")
def step_then_different_sandbox(context: Any) -> None:
for key, ref_sandbox in context._sandbox_refs.items():
_plan_id, _res_id = key.split(":")
if (
context.sandbox_result is not None
and context.sandbox_result is not ref_sandbox
):
return # Found a different instance
raise AssertionError("Expected a different sandbox instance")
@then('{count:d} sandboxes should be tracked for plan "{plan_id}"')
def step_then_sandbox_count(context: Any, count: int, plan_id: str) -> None:
tracked = context.manager._active_sandboxes.get(plan_id, {})
assert len(tracked) == count, (
f"Expected {count} sandboxes for plan '{plan_id}', got {len(tracked)}"
)
@then("the sandbox result should be None")
def step_then_sandbox_result_none(context: Any) -> None:
assert context.sandbox_result is None, (
f"Expected None, got {context.sandbox_result}"
)
@then("the sandbox list should contain {count:d} sandboxes")
def step_then_sandbox_list_count(context: Any, count: int) -> None:
assert len(context.sandbox_list) == count, (
f"Expected {count} sandboxes in list, got {len(context.sandbox_list)}"
)
@then("{count:d} commit results should be returned")
def step_then_commit_results_count(context: Any, count: int) -> None:
assert context.commit_results is not None, "No commit results available"
assert len(context.commit_results) == count, (
f"Expected {count} commit results, got {len(context.commit_results)}"
)
@then("all commit results should be successful")
def step_then_all_commits_successful(context: Any) -> None:
for result in context.commit_results:
assert result.success is True, (
f"Expected successful commit, got error: {result.error}"
)
@then("the first commit result should have success false")
def step_then_first_commit_failed(context: Any) -> None:
assert context.commit_results is not None and len(context.commit_results) > 0
assert context.commit_results[0].success is False, (
"Expected first commit result to be unsuccessful"
)
@then("no sandbox error should be raised")
def step_then_no_sandbox_error(context: Any) -> None:
assert context.error is None, f"Unexpected error: {context.error}"
@then('plan "{plan_id}" should no longer be tracked')
def step_then_plan_not_tracked(context: Any, plan_id: str) -> None:
assert plan_id not in context.manager._active_sandboxes, (
f"Plan '{plan_id}' is still tracked in active sandboxes"
)
@then("{count:d} abandoned sandboxes should be cleaned up")
def step_then_abandoned_count(context: Any, count: int) -> None:
assert context.abandoned_count == count, (
f"Expected {count} abandoned sandboxes cleaned, got {context.abandoned_count}"
)