"""Helper script for Robot Framework semantic escalation tests.""" from __future__ import annotations import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.application.services.autonomy_controller import ( AutonomyController, ) from cleveragents.domain.models.core.automation_profile import ( BUILTIN_PROFILES, AutomationProfile, ) from cleveragents.domain.models.core.escalation import ( ConfidenceFactors, HistoricalOutcome, OperationContext, ) def _test_confidence_high() -> None: """All positive factors produce high confidence.""" ctrl = AutonomyController() factors = ConfidenceFactors( past_success_rate=1.0, codebase_familiarity=1.0, risk_assessment=0.0, invariant_complexity=0.0, ) score = ctrl.compute_confidence(factors) assert abs(score - 1.0) < 1e-6, f"Expected 1.0, got {score}" print("confidence-high-ok") def _test_confidence_low() -> None: """All negative factors produce zero confidence.""" ctrl = AutonomyController() factors = ConfidenceFactors( past_success_rate=0.0, codebase_familiarity=0.0, risk_assessment=1.0, invariant_complexity=1.0, ) score = ctrl.compute_confidence(factors) assert abs(score - 0.0) < 1e-6, f"Expected 0.0, got {score}" print("confidence-low-ok") def _test_threshold_proceed() -> None: """Proceed when confidence >= threshold.""" ctrl = AutonomyController() profile = AutomationProfile(name="test", create_tool=0.5) factors = ConfidenceFactors( past_success_rate=0.9, codebase_familiarity=0.9, risk_assessment=0.1, invariant_complexity=0.1, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.proceed is True, f"Expected proceed=True, got {decision.proceed}" assert decision.confidence >= 0.5 print("threshold-proceed-ok") def _test_threshold_escalate() -> None: """Escalate when confidence < threshold.""" ctrl = AutonomyController() profile = AutomationProfile(name="test", create_tool=0.99) factors = ConfidenceFactors( past_success_rate=0.3, codebase_familiarity=0.3, risk_assessment=0.7, invariant_complexity=0.7, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.proceed is False, f"Expected proceed=False, got {decision.proceed}" print("threshold-escalate-ok") def _test_history_tracking() -> None: """Success rate updates after recording outcomes.""" ctrl = AutonomyController() assert abs(ctrl.get_historical_success_rate("create_tool") - 0.5) < 1e-6 for _ in range(8): ctrl.record_outcome( HistoricalOutcome(operation_type="create_tool", succeeded=True) ) for _ in range(2): ctrl.record_outcome( HistoricalOutcome(operation_type="create_tool", succeeded=False) ) rate = ctrl.get_historical_success_rate("create_tool") assert abs(rate - 0.8) < 1e-6, f"Expected 0.8, got {rate}" assert ctrl.get_history_count("create_tool") == 10 print("history-tracking-ok") def _test_profile_manual() -> None: """Manual profile always escalates (with realistic confidence < 1.0).""" ctrl = AutonomyController() profile = BUILTIN_PROFILES["manual"] factors = ConfidenceFactors( past_success_rate=0.95, codebase_familiarity=0.95, risk_assessment=0.05, invariant_complexity=0.05, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.proceed is False print("profile-manual-ok") def _test_profile_fullauto() -> None: """Full-auto profile always proceeds.""" ctrl = AutonomyController() profile = BUILTIN_PROFILES["full-auto"] factors = ConfidenceFactors( past_success_rate=0.0, codebase_familiarity=0.0, risk_assessment=1.0, invariant_complexity=1.0, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.proceed is True print("profile-fullauto-ok") def _test_profile_cautious() -> None: """Cautious profile uses intermediate thresholds.""" ctrl = AutonomyController() profile = BUILTIN_PROFILES["cautious"] # High confidence should proceed high_factors = ConfidenceFactors( past_success_rate=0.95, codebase_familiarity=0.9, risk_assessment=0.05, invariant_complexity=0.05, ) op = OperationContext(operation_type="create_tool") high_decision = ctrl.should_proceed_automatically( op, high_factors, profile, ) assert high_decision.proceed is True # Low confidence should escalate low_factors = ConfidenceFactors( past_success_rate=0.3, codebase_familiarity=0.3, risk_assessment=0.8, invariant_complexity=0.8, ) low_decision = ctrl.should_proceed_automatically( op, low_factors, profile, ) assert low_decision.proceed is False print("profile-cautious-ok") def _test_decision_fields() -> None: """Decision model includes all required fields.""" ctrl = AutonomyController() profile = AutomationProfile(name="test", create_tool=0.5) factors = ConfidenceFactors( past_success_rate=0.8, codebase_familiarity=0.7, risk_assessment=0.2, invariant_complexity=0.3, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.operation_type == "create_tool" assert len(decision.factors) == 4 assert "past_success_rate" in decision.factors assert abs(decision.threshold - 0.5) < 1e-6 assert isinstance(decision.explanation, str) assert len(decision.explanation) > 0 print("decision-fields-ok") def _test_zero_threshold() -> None: """Threshold 0.0 means always automatic, even zero confidence.""" ctrl = AutonomyController() profile = AutomationProfile(name="test", create_tool=0.0) factors = ConfidenceFactors( past_success_rate=0.0, codebase_familiarity=0.0, risk_assessment=1.0, invariant_complexity=1.0, ) op = OperationContext(operation_type="create_tool") decision = ctrl.should_proceed_automatically(op, factors, profile) assert decision.proceed is True assert abs(decision.confidence - 0.0) < 1e-6 print("zero-threshold-ok") _TESTS = { "confidence-high": _test_confidence_high, "confidence-low": _test_confidence_low, "threshold-proceed": _test_threshold_proceed, "threshold-escalate": _test_threshold_escalate, "history-tracking": _test_history_tracking, "profile-manual": _test_profile_manual, "profile-fullauto": _test_profile_fullauto, "profile-cautious": _test_profile_cautious, "decision-fields": _test_decision_fields, "zero-threshold": _test_zero_threshold, } if __name__ == "__main__": if len(sys.argv) < 2: print(f"Usage: {sys.argv[0]} ", file=sys.stderr) print(f"Available tests: {', '.join(sorted(_TESTS))}", file=sys.stderr) sys.exit(1) test_name = sys.argv[1] if test_name not in _TESTS: print(f"Unknown test: {test_name}", file=sys.stderr) sys.exit(1) _TESTS[test_name]()