cleveragents-core/robot/helper_semantic_escalation.py

"""Helper script for Robot Framework semantic escalation tests."""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from cleveragents.application.services.autonomy_controller import (
    AutonomyController,
)
from cleveragents.domain.models.core.automation_profile import (
    BUILTIN_PROFILES,
    AutomationProfile,
)
from cleveragents.domain.models.core.escalation import (
    ConfidenceFactors,
    HistoricalOutcome,
    OperationContext,
)


def _test_confidence_high() -> None:
    """All positive factors produce high confidence."""
    ctrl = AutonomyController()
    factors = ConfidenceFactors(
        past_success_rate=1.0,
        codebase_familiarity=1.0,
        risk_assessment=0.0,
        invariant_complexity=0.0,
    )
    score = ctrl.compute_confidence(factors)
    assert abs(score - 1.0) < 1e-6, f"Expected 1.0, got {score}"
    print("confidence-high-ok")


def _test_confidence_low() -> None:
    """All negative factors produce zero confidence."""
    ctrl = AutonomyController()
    factors = ConfidenceFactors(
        past_success_rate=0.0,
        codebase_familiarity=0.0,
        risk_assessment=1.0,
        invariant_complexity=1.0,
    )
    score = ctrl.compute_confidence(factors)
    assert abs(score - 0.0) < 1e-6, f"Expected 0.0, got {score}"
    print("confidence-low-ok")


def _test_threshold_proceed() -> None:
    """Proceed when confidence >= threshold."""
    ctrl = AutonomyController()
    profile = AutomationProfile(name="test", create_tool=0.5)
    factors = ConfidenceFactors(
        past_success_rate=0.9,
        codebase_familiarity=0.9,
        risk_assessment=0.1,
        invariant_complexity=0.1,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)
    assert decision.proceed is True, f"Expected proceed=True, got {decision.proceed}"
    assert decision.confidence >= 0.5
    print("threshold-proceed-ok")


def _test_threshold_escalate() -> None:
    """Escalate when confidence < threshold."""
    ctrl = AutonomyController()
    profile = AutomationProfile(name="test", create_tool=0.99)
    factors = ConfidenceFactors(
        past_success_rate=0.3,
        codebase_familiarity=0.3,
        risk_assessment=0.7,
        invariant_complexity=0.7,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)
    assert decision.proceed is False, f"Expected proceed=False, got {decision.proceed}"
    print("threshold-escalate-ok")


def _test_history_tracking() -> None:
    """Success rate updates after recording outcomes."""
    ctrl = AutonomyController()
    assert abs(ctrl.get_historical_success_rate("create_tool") - 0.5) < 1e-6

    for _ in range(8):
        ctrl.record_outcome(
            HistoricalOutcome(operation_type="create_tool", succeeded=True)
        )
    for _ in range(2):
        ctrl.record_outcome(
            HistoricalOutcome(operation_type="create_tool", succeeded=False)
        )

    rate = ctrl.get_historical_success_rate("create_tool")
    assert abs(rate - 0.8) < 1e-6, f"Expected 0.8, got {rate}"
    assert ctrl.get_history_count("create_tool") == 10
    print("history-tracking-ok")


def _test_profile_manual() -> None:
    """Manual profile always escalates (with realistic confidence < 1.0)."""
    ctrl = AutonomyController()
    profile = BUILTIN_PROFILES["manual"]
    factors = ConfidenceFactors(
        past_success_rate=0.95,
        codebase_familiarity=0.95,
        risk_assessment=0.05,
        invariant_complexity=0.05,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)
    assert decision.proceed is False
    print("profile-manual-ok")


def _test_profile_fullauto() -> None:
    """Full-auto profile always proceeds."""
    ctrl = AutonomyController()
    profile = BUILTIN_PROFILES["full-auto"]
    factors = ConfidenceFactors(
        past_success_rate=0.0,
        codebase_familiarity=0.0,
        risk_assessment=1.0,
        invariant_complexity=1.0,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)
    assert decision.proceed is True
    print("profile-fullauto-ok")


def _test_profile_cautious() -> None:
    """Cautious profile uses intermediate thresholds."""
    ctrl = AutonomyController()
    profile = BUILTIN_PROFILES["cautious"]

    # High confidence should proceed
    high_factors = ConfidenceFactors(
        past_success_rate=0.95,
        codebase_familiarity=0.9,
        risk_assessment=0.05,
        invariant_complexity=0.05,
    )
    op = OperationContext(operation_type="create_tool")
    high_decision = ctrl.should_proceed_automatically(
        op,
        high_factors,
        profile,
    )
    assert high_decision.proceed is True

    # Low confidence should escalate
    low_factors = ConfidenceFactors(
        past_success_rate=0.3,
        codebase_familiarity=0.3,
        risk_assessment=0.8,
        invariant_complexity=0.8,
    )
    low_decision = ctrl.should_proceed_automatically(
        op,
        low_factors,
        profile,
    )
    assert low_decision.proceed is False
    print("profile-cautious-ok")


def _test_decision_fields() -> None:
    """Decision model includes all required fields."""
    ctrl = AutonomyController()
    profile = AutomationProfile(name="test", create_tool=0.5)
    factors = ConfidenceFactors(
        past_success_rate=0.8,
        codebase_familiarity=0.7,
        risk_assessment=0.2,
        invariant_complexity=0.3,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)

    assert decision.operation_type == "create_tool"
    assert len(decision.factors) == 4
    assert "past_success_rate" in decision.factors
    assert abs(decision.threshold - 0.5) < 1e-6
    assert isinstance(decision.explanation, str)
    assert len(decision.explanation) > 0
    print("decision-fields-ok")


def _test_zero_threshold() -> None:
    """Threshold 0.0 means always automatic, even zero confidence."""
    ctrl = AutonomyController()
    profile = AutomationProfile(name="test", create_tool=0.0)
    factors = ConfidenceFactors(
        past_success_rate=0.0,
        codebase_familiarity=0.0,
        risk_assessment=1.0,
        invariant_complexity=1.0,
    )
    op = OperationContext(operation_type="create_tool")
    decision = ctrl.should_proceed_automatically(op, factors, profile)
    assert decision.proceed is True
    assert abs(decision.confidence - 0.0) < 1e-6
    print("zero-threshold-ok")


_TESTS = {
    "confidence-high": _test_confidence_high,
    "confidence-low": _test_confidence_low,
    "threshold-proceed": _test_threshold_proceed,
    "threshold-escalate": _test_threshold_escalate,
    "history-tracking": _test_history_tracking,
    "profile-manual": _test_profile_manual,
    "profile-fullauto": _test_profile_fullauto,
    "profile-cautious": _test_profile_cautious,
    "decision-fields": _test_decision_fields,
    "zero-threshold": _test_zero_threshold,
}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print(f"Usage: {sys.argv[0]} <test-name>", file=sys.stderr)
        print(f"Available tests: {', '.join(sorted(_TESTS))}", file=sys.stderr)
        sys.exit(1)

    test_name = sys.argv[1]
    if test_name not in _TESTS:
        print(f"Unknown test: {test_name}", file=sys.stderr)
        sys.exit(1)

    _TESTS[test_name]()