cleveragents-core/robot/helper_m6_autonomy_acceptance.py

"""Helper script for m6_autonomy_acceptance.robot E2E tests.

Each subcommand is a self-contained check that prints a sentinel on success.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

# Ensure local source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
    sys.path.insert(0, _SRC)

from cleveragents.a2a.errors import (  # noqa: E402
    A2aNotAvailableError,
    A2aOperationNotFoundError,
    A2aVersionMismatchError,
)
from cleveragents.a2a.events import A2aEventQueue  # noqa: E402
from cleveragents.a2a.facade import A2aLocalFacade  # noqa: E402
from cleveragents.a2a.models import A2aEvent, A2aRequest  # noqa: E402
from cleveragents.a2a.transport import A2aHttpTransport  # noqa: E402
from cleveragents.a2a.versioning import A2aVersionNegotiator  # noqa: E402
from cleveragents.application.services.automation_profile_service import (  # noqa: E402
    AutomationProfileService,
)
from cleveragents.domain.models.core.automation_guard import (  # noqa: E402
    AutomationGuard,
)
from cleveragents.domain.models.core.automation_profile import (  # noqa: E402
    AutomationProfile,
)

_FIXTURES_DIR = Path(__file__).resolve().parents[1] / "features" / "fixtures" / "m6"


# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------


def facade_session() -> None:
    """Dispatch session.create and session.close."""
    facade = A2aLocalFacade()

    resp_create = facade.dispatch(A2aRequest(method="session.create", params={}))
    assert resp_create.result is not None, f"Expected ok, got {resp_create.status}"
    assert "session_id" in resp_create.result

    resp_close = facade.dispatch(A2aRequest(method="session.close", params={}))
    assert resp_close.result is not None
    assert resp_close.result["status"] == "closed"

    print("m6-facade-session-ok")


def facade_plan() -> None:
    """Dispatch plan create/execute/status/diff/apply."""
    facade = A2aLocalFacade()

    resp = facade.dispatch(A2aRequest(method="plan.create", params={}))
    assert resp.result is not None
    plan_id = resp.result["plan_id"]
    assert plan_id

    resp = facade.dispatch(
        A2aRequest(method="plan.execute", params={"plan_id": plan_id})
    )
    assert resp.result is not None
    assert resp.result["status"] == "queued"

    resp = facade.dispatch(
        A2aRequest(method="plan.status", params={"plan_id": plan_id})
    )
    assert resp.result is not None
    assert "phase" in resp.result

    resp = facade.dispatch(A2aRequest(method="plan.diff", params={"plan_id": plan_id}))
    assert resp.result is not None
    assert "changes" in resp.result

    resp = facade.dispatch(A2aRequest(method="plan.apply", params={"plan_id": plan_id}))
    assert resp.result is not None
    assert resp.result["status"] == "applied"

    print("m6-facade-plan-ok")


def facade_unknown_op() -> None:
    """Verify unknown operation raises A2aOperationNotFoundError."""
    facade = A2aLocalFacade()
    try:
        facade.dispatch(A2aRequest(method="nonexistent.op", params={}))
        print("FAIL: expected A2aOperationNotFoundError")
        sys.exit(1)
    except A2aOperationNotFoundError:
        pass

    print("m6-facade-unknown-op-ok")


def event_queue() -> None:
    """Publish events and verify local subscriber receives them."""
    queue = A2aEventQueue()
    received = []

    sub_id = queue.subscribe_local(lambda e: received.append(e))
    queue.publish(A2aEvent(event_type="plan.progress", data={"step": 1}))
    assert len(received) == 1
    assert received[0].event_type == "plan.progress"

    # Unsubscribe and verify no more callbacks
    queue.unsubscribe(sub_id)
    queue.publish(A2aEvent(event_type="plan.complete", data={}))
    assert len(received) == 1  # still 1

    # Verify get_events returns both
    events = queue.get_events()
    assert len(events) == 2

    # Close and verify publish raises
    queue.close()
    try:
        queue.publish(A2aEvent(event_type="after.close", data={}))
        print("FAIL: expected RuntimeError")
        sys.exit(1)
    except RuntimeError:
        pass

    print("m6-event-queue-ok")


def transport_stub() -> None:
    """Verify HTTP transport stub raises A2aNotAvailableError."""
    transport = A2aHttpTransport()

    # send
    try:
        transport.send(A2aRequest(method="plan.create", params={}))
        print("FAIL: expected A2aNotAvailableError on send")
        sys.exit(1)
    except A2aNotAvailableError:
        pass

    # connect
    try:
        transport.connect("https://example.com/a2a")
        print("FAIL: expected A2aNotAvailableError on connect")
        sys.exit(1)
    except A2aNotAvailableError:
        pass

    # disconnect
    try:
        transport.disconnect()
        print("FAIL: expected A2aNotAvailableError on disconnect")
        sys.exit(1)
    except A2aNotAvailableError:
        pass

    # is_connected
    assert transport.is_connected() is False

    print("m6-transport-stub-ok")


def version_negotiation() -> None:
    """Negotiate supported and unsupported A2A versions."""
    negotiator = A2aVersionNegotiator()

    # Supported
    result = negotiator.negotiate("1.0")
    assert result == "1.0"
    assert negotiator.is_supported("1.0") is True
    assert negotiator.get_current() == "1.0"

    # Unsupported
    try:
        negotiator.negotiate("2.0")
        print("FAIL: expected A2aVersionMismatchError")
        sys.exit(1)
    except A2aVersionMismatchError:
        pass

    assert negotiator.is_supported("99.0") is False

    print("m6-version-negotiation-ok")


def guard_denylist() -> None:
    """Verify denylist guard blocks denied tools."""
    guard = AutomationGuard(tool_denylist=["rm_rf", "drop_database"])
    profile = AutomationProfile(name="test-deny", guards=guard)

    # Allowed tool
    result = profile.check_guard(tool_name="read_file")
    assert result.allowed is True

    # Denied tool
    result = profile.check_guard(tool_name="rm_rf")
    assert result.allowed is False
    assert result.requires_approval is True
    assert "denylist" in (result.reason or "")

    print("m6-guard-denylist-ok")


def guard_budget() -> None:
    """Verify cost budget and call limit guards."""
    guard = AutomationGuard(max_total_cost=10.0, max_tool_calls_per_step=5)
    profile = AutomationProfile(name="test-budget", guards=guard)

    # Under budget and limit
    result = profile.check_guard(tool_name="llm_call", cost_so_far=3.0, calls_so_far=2)
    assert result.allowed is True

    # Over budget
    result = profile.check_guard(tool_name="llm_call", cost_so_far=10.0, calls_so_far=2)
    assert result.allowed is False
    assert (
        "budget" in (result.reason or "").lower()
        or "cost" in (result.reason or "").lower()
    )

    # Over call limit
    result = profile.check_guard(tool_name="llm_call", cost_so_far=3.0, calls_so_far=5)
    assert result.allowed is False
    assert "limit" in (result.reason or "").lower()

    print("m6-guard-budget-ok")


def profile_resolution() -> None:
    """Verify plan > action > project > global resolution."""
    service = AutomationProfileService()

    # Plan takes precedence
    p = service.resolve_profile(
        plan_profile="ci", action_profile="auto", project_profile="manual"
    )
    assert p.name == "ci"

    # Action takes precedence over project
    p = service.resolve_profile(
        plan_profile=None, action_profile="auto", project_profile="manual"
    )
    assert p.name == "auto"

    # Project takes precedence over global
    p = service.resolve_profile(
        plan_profile=None, action_profile=None, project_profile="review"
    )
    assert p.name == "review"

    # Falls back to global default (manual)
    p = service.resolve_profile(
        plan_profile=None, action_profile=None, project_profile=None
    )
    assert p.name == "manual"

    print("m6-profile-resolution-ok")


def fixture_loading() -> None:
    """Load all M6 fixture files and verify structure."""
    for fname in (
        "a2a_facade_flows.json",
        "autonomy_guardrails.json",
        "automation_profiles.json",
    ):
        fpath = _FIXTURES_DIR / fname
        with open(fpath) as f:
            data = json.load(f)
        assert "fixtures" in data, f"Missing 'fixtures' key in {fname}"
        assert len(data["fixtures"]) > 0, f"Empty fixtures in {fname}"
    print("m6-fixture-loading-ok")


def full_flow() -> None:
    """End-to-end: facade dispatch + guard check + profile resolution."""
    # Step 1: Facade dispatch
    facade = A2aLocalFacade()
    resp = facade.dispatch(A2aRequest(method="session.create", params={}))
    assert resp.result is not None

    resp = facade.dispatch(A2aRequest(method="plan.create", params={}))
    assert resp.result is not None
    plan_id = resp.result["plan_id"]

    resp = facade.dispatch(
        A2aRequest(method="plan.execute", params={"plan_id": plan_id})
    )
    assert resp.result is not None

    # Step 2: Guard check
    guard = AutomationGuard(
        tool_denylist=["dangerous_tool"],
        max_tool_calls_per_step=3,
    )
    profile = AutomationProfile(name="test-full-flow", guards=guard)

    result = profile.check_guard(tool_name="safe_tool", calls_so_far=1)
    assert result.allowed is True

    result = profile.check_guard(tool_name="dangerous_tool")
    assert result.allowed is False

    # Step 3: Profile resolution
    service = AutomationProfileService()
    p = service.resolve_profile(plan_profile="ci")
    assert p.name == "ci"

    # Step 4: Version negotiation
    negotiator = A2aVersionNegotiator()
    assert negotiator.negotiate("1.0") == "1.0"

    # Step 5: Event queue
    queue = A2aEventQueue()
    queue.publish(A2aEvent(event_type="flow.complete", data={}))
    assert len(queue.get_events()) == 1
    queue.close()

    print("m6-full-flow-ok")


# ---------------------------------------------------------------------------
# Dispatcher
# ---------------------------------------------------------------------------

_COMMANDS: dict[str, object] = {
    "facade-session": facade_session,
    "facade-plan": facade_plan,
    "facade-unknown-op": facade_unknown_op,
    "event-queue": event_queue,
    "transport-stub": transport_stub,
    "version-negotiation": version_negotiation,
    "guard-denylist": guard_denylist,
    "guard-budget": guard_budget,
    "profile-resolution": profile_resolution,
    "fixture-loading": fixture_loading,
    "full-flow": full_flow,
}

if __name__ == "__main__":
    if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
        print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>")
        sys.exit(1)
    fn = _COMMANDS[sys.argv[1]]
    fn()  # type: ignore[operator]