"""Step definitions for Semantic Escalation with Confidence Scoring.""" from __future__ import annotations from behave import given, then, when from behave.runner import Context from cleveragents.application.services.autonomy_controller import ( AutonomyController, ) from cleveragents.domain.models.core.automation_profile import ( BUILTIN_PROFILES, AutomationProfile, ) from cleveragents.domain.models.core.escalation import ( ConfidenceFactors, HistoricalOutcome, OperationContext, ) # ---- Background / Setup steps ---- @given("a default autonomy controller") def step_default_controller(context: Context) -> None: """Create a default AutonomyController instance.""" context.controller = AutonomyController() context.decision = None context.error = None # ---- Confidence computation steps ---- @when( "I compute confidence with past_success_rate {psr:g} " "codebase_familiarity {cf:g} risk_assessment {ra:g} " "invariant_complexity {ic:g}" ) def step_compute_confidence( context: Context, psr: float, cf: float, ra: float, ic: float, ) -> None: """Compute confidence from provided factors.""" factors = ConfidenceFactors( past_success_rate=psr, codebase_familiarity=cf, risk_assessment=ra, invariant_complexity=ic, ) context.confidence = context.controller.compute_confidence(factors) @then("the confidence score should be {expected:g}") def step_confidence_should_be(context: Context, expected: float) -> None: """Assert exact confidence score.""" assert abs(context.confidence - expected) < 1e-6, ( f"Expected confidence {expected}, got {context.confidence}" ) @then("the confidence score should be approximately {expected:g}") def step_confidence_approximately( context: Context, expected: float, ) -> None: """Assert confidence within tolerance.""" assert abs(context.confidence - expected) < 0.02, ( f"Expected confidence ~{expected}, got {context.confidence}" ) @then("the confidence score should be less than {bound:g}") def step_confidence_less_than(context: Context, bound: float) -> None: """Assert confidence is strictly less than bound.""" assert context.confidence < bound, ( f"Expected confidence < {bound}, got {context.confidence}" ) # ---- Threshold / Profile steps ---- @given('the automation profile "{profile_name}"') def step_given_profile(context: Context, profile_name: str) -> None: """Load a built-in automation profile by name.""" context.profile = BUILTIN_PROFILES[profile_name] @given("a profile with create_tool threshold {threshold:g}") def step_given_custom_threshold_profile( context: Context, threshold: float, ) -> None: """Create a minimal profile with a specific create_tool threshold.""" context.profile = AutomationProfile( name="test-profile", create_tool=threshold, ) @when( 'I evaluate escalation for operation "{op}" with confidence factors ' "past_success_rate {psr:g} codebase_familiarity {cf:g} " "risk_assessment {ra:g} invariant_complexity {ic:g}" ) def step_evaluate_escalation( context: Context, op: str, psr: float, cf: float, ra: float, ic: float, ) -> None: """Evaluate should_proceed_automatically with given factors.""" factors = ConfidenceFactors( past_success_rate=psr, codebase_familiarity=cf, risk_assessment=ra, invariant_complexity=ic, ) operation = OperationContext(operation_type=op) context.decision = context.controller.should_proceed_automatically( operation=operation, factors=factors, profile=context.profile, ) context.confidence = context.decision.confidence @then( 'the escalation decision should match the "{profile_name}" ' 'profile threshold for "{op}"' ) def step_decision_matches_profile( context: Context, profile_name: str, op: str, ) -> None: """Verify the decision matches the profile's threshold for the op.""" profile = BUILTIN_PROFILES[profile_name] threshold = getattr(profile, op, 1.0) decision = context.decision assert decision is not None if threshold == 0.0: assert decision.proceed is True, ( f"Profile '{profile_name}' has threshold 0.0 for '{op}', " f"expected proceed=True but got False" ) elif threshold == 1.0: assert decision.proceed is False, ( f"Profile '{profile_name}' has threshold 1.0 for '{op}', " f"expected proceed=False but got True" ) else: expected = decision.confidence >= threshold assert decision.proceed == expected, ( f"Profile '{profile_name}': confidence={decision.confidence:.3f} " f"threshold={threshold:.3f}, expected proceed={expected} " f"but got {decision.proceed}" ) @then("the escalation decision proceed should be true") def step_decision_proceed_true(context: Context) -> None: """Assert the escalation decision is to proceed.""" assert context.decision is not None assert context.decision.proceed is True, ( f"Expected proceed=True, got False " f"(confidence={context.decision.confidence:.3f}, " f"threshold={context.decision.threshold:.3f})" ) @then("the escalation decision proceed should be false") def step_decision_proceed_false(context: Context) -> None: """Assert the escalation decision is to escalate.""" assert context.decision is not None assert context.decision.proceed is False, ( f"Expected proceed=False, got True " f"(confidence={context.decision.confidence:.3f}, " f"threshold={context.decision.threshold:.3f})" ) @then("the escalation threshold should be {expected:g}") def step_escalation_threshold(context: Context, expected: float) -> None: """Assert the threshold used in the decision.""" assert context.decision is not None assert abs(context.decision.threshold - expected) < 1e-6, ( f"Expected threshold {expected}, got {context.decision.threshold}" ) @then("the confidence should be {expected:g}") def step_confidence_exact(context: Context, expected: float) -> None: """Assert confidence from the decision.""" assert context.decision is not None assert abs(context.decision.confidence - expected) < 1e-6, ( f"Expected confidence {expected}, got {context.decision.confidence}" ) # ---- Historical success tracking steps ---- @then('the historical success rate for "{op}" should be {expected:g}') def step_historical_rate( context: Context, op: str, expected: float, ) -> None: """Assert historical success rate.""" rate = context.controller.get_historical_success_rate(op) assert abs(rate - expected) < 1e-6, ( f"Expected rate {expected} for '{op}', got {rate}" ) @then('the historical success rate for "{op}" should be approximately {expected:g}') def step_historical_rate_approx( context: Context, op: str, expected: float, ) -> None: """Assert historical success rate within tolerance.""" rate = context.controller.get_historical_success_rate(op) assert abs(rate - expected) < 0.02, ( f"Expected rate ~{expected} for '{op}', got {rate}" ) @when('I record {successes:d} successes and {failures:d} failures for "{op}"') def step_record_outcomes( context: Context, successes: int, failures: int, op: str, ) -> None: """Record a batch of successes and failures.""" for _ in range(successes): context.controller.record_outcome( HistoricalOutcome(operation_type=op, succeeded=True) ) for _ in range(failures): context.controller.record_outcome( HistoricalOutcome(operation_type=op, succeeded=False) ) @when('I record {count:d} additional successes for "{op}"') def step_record_additional_successes( context: Context, count: int, op: str, ) -> None: """Record additional successes.""" for _ in range(count): context.controller.record_outcome( HistoricalOutcome(operation_type=op, succeeded=True) ) @when('I clear history for "{op}"') def step_clear_history(context: Context, op: str) -> None: """Clear history for a specific operation type.""" context.controller.clear_history(op) @then('the history count for "{op}" should be {expected:d}') def step_history_count( context: Context, op: str, expected: int, ) -> None: """Assert the number of recorded outcomes.""" count = context.controller.get_history_count(op) assert count == expected, f"Expected {expected} outcomes for '{op}', got {count}" # ---- Explanation steps ---- @then('the escalation explanation should contain "{text}"') def step_explanation_contains(context: Context, text: str) -> None: """Assert the explanation contains expected text.""" assert context.decision is not None assert text in context.decision.explanation, ( f"Expected '{text}' in explanation: {context.decision.explanation}" ) # ---- Custom weights steps ---- @when("I try to create a controller with invalid weights") def step_try_invalid_weights(context: Context) -> None: """Attempt to create a controller with bad weights.""" try: AutonomyController( weights={ "past_success_rate": 0.5, "codebase_familiarity": 0.5, # missing risk_assessment and invariant_complexity } ) context.error = None except ValueError as exc: context.error = exc @then("a weight validation error should be raised") def step_weight_validation_error_raised(context: Context) -> None: """Assert a ValueError was raised for weight validation.""" assert context.error is not None assert isinstance(context.error, ValueError), ( f"Expected ValueError, got {type(context.error).__name__}" ) @given("a controller with equal weights") def step_equal_weights_controller(context: Context) -> None: """Create a controller where all 4 weights are 0.25.""" context.controller = AutonomyController( weights={ "past_success_rate": 0.25, "codebase_familiarity": 0.25, "risk_assessment": 0.25, "invariant_complexity": 0.25, } ) # ---- Model validation steps ---- @when("I try to create confidence factors with past_success_rate {value:g}") def step_try_bad_confidence_factors( context: Context, value: float, ) -> None: """Attempt to create ConfidenceFactors with out-of-range value.""" try: ConfidenceFactors(past_success_rate=value) context.error = None except Exception as exc: context.error = exc @then("a factor validation error should be raised") def step_factor_validation_error_raised(context: Context) -> None: """Assert a validation error was raised for factors.""" assert context.error is not None, "Expected an error but none was raised" @then("an operation context validation error should be raised") def step_op_context_validation_error_raised(context: Context) -> None: """Assert a validation error was raised for operation context.""" assert context.error is not None, "Expected an error but none was raised" @when("I try to create an operation context with empty operation type") def step_try_empty_operation_context(context: Context) -> None: """Attempt to create OperationContext with empty operation_type.""" try: OperationContext(operation_type="") context.error = None except Exception as exc: context.error = exc # ---- EscalationDecision field steps ---- @then('the escalation decision should have operation_type "{op}"') def step_decision_operation_type(context: Context, op: str) -> None: """Assert the decision's operation_type.""" assert context.decision is not None assert context.decision.operation_type == op @then("the escalation decision should have {count:d} factors") def step_decision_factor_count(context: Context, count: int) -> None: """Assert the number of factors in the decision.""" assert context.decision is not None assert len(context.decision.factors) == count, ( f"Expected {count} factors, got {len(context.decision.factors)}" ) @then("the escalation decision threshold should be {expected:g}") def step_decision_threshold_value( context: Context, expected: float, ) -> None: """Assert the decision's threshold.""" assert context.decision is not None assert abs(context.decision.threshold - expected) < 1e-6, ( f"Expected threshold {expected}, got {context.decision.threshold}" ) # ---- Argument validation coverage steps ---- @when("I call should_proceed_automatically with None operation") def step_call_proceed_none_operation(context: Context) -> None: """Call should_proceed_automatically with None operation.""" try: factors = ConfidenceFactors() profile = AutomationProfile(name="test") context.controller.should_proceed_automatically( None, factors, profile, # type: ignore[arg-type] ) context.error = None except ValueError as exc: context.error = exc @when("I call should_proceed_automatically with None factors") def step_call_proceed_none_factors(context: Context) -> None: """Call should_proceed_automatically with None factors.""" try: op = OperationContext(operation_type="create_tool") profile = AutomationProfile(name="test") context.controller.should_proceed_automatically( op, None, profile, # type: ignore[arg-type] ) context.error = None except ValueError as exc: context.error = exc @when("I call should_proceed_automatically with None profile") def step_call_proceed_none_profile(context: Context) -> None: """Call should_proceed_automatically with None profile.""" try: op = OperationContext(operation_type="create_tool") factors = ConfidenceFactors() context.controller.should_proceed_automatically( op, factors, None, # type: ignore[arg-type] ) context.error = None except ValueError as exc: context.error = exc @when("I call compute_confidence with None factors") def step_call_compute_confidence_none(context: Context) -> None: """Call compute_confidence with None factors.""" try: context.controller.compute_confidence( None, # type: ignore[arg-type] ) context.error = None except ValueError as exc: context.error = exc @when("I call record_outcome with None outcome") def step_call_record_outcome_none(context: Context) -> None: """Call record_outcome with None outcome.""" try: context.controller.record_outcome( None, # type: ignore[arg-type] ) context.error = None except ValueError as exc: context.error = exc @when("I call get_historical_success_rate with empty string") def step_call_get_rate_empty(context: Context) -> None: """Call get_historical_success_rate with empty string.""" try: context.controller.get_historical_success_rate("") context.error = None except ValueError as exc: context.error = exc @when("I call get_history_count with empty string") def step_call_get_count_empty(context: Context) -> None: """Call get_history_count with empty string.""" try: context.controller.get_history_count("") context.error = None except ValueError as exc: context.error = exc @when("I clear all history") def step_clear_all_history(context: Context) -> None: """Clear all history.""" context.controller.clear_history() @then("the controller weights should have {count:d} entries") def step_weights_count(context: Context, count: int) -> None: """Assert the number of weight entries.""" weights = context.controller.weights assert len(weights) == count, f"Expected {count} weights, got {len(weights)}" @when('I record max history outcomes for "{op}"') def step_record_max_history(context: Context, op: str) -> None: """Record exactly _MAX_HISTORY_PER_TYPE outcomes to trigger eviction.""" from cleveragents.application.services.autonomy_controller import ( _MAX_HISTORY_PER_TYPE, ) for _ in range(_MAX_HISTORY_PER_TYPE + 1): context.controller.record_outcome( HistoricalOutcome(operation_type=op, succeeded=True) ) @when("I try to create a controller with extra weight keys") def step_try_extra_weight_keys(context: Context) -> None: """Attempt to create controller with extra weight keys.""" try: AutonomyController( weights={ "past_success_rate": 0.2, "codebase_familiarity": 0.2, "risk_assessment": 0.2, "invariant_complexity": 0.2, "extra_key": 0.2, } ) context.error = None except ValueError as exc: context.error = exc @when("I try to create a controller with negative weight") def step_try_negative_weight(context: Context) -> None: """Attempt to create controller with negative weight.""" try: AutonomyController( weights={ "past_success_rate": -0.1, "codebase_familiarity": 0.4, "risk_assessment": 0.4, "invariant_complexity": 0.3, } ) context.error = None except ValueError as exc: context.error = exc @when("I try to create a controller with weights not summing to 1") def step_try_bad_sum_weights(context: Context) -> None: """Attempt to create controller with weights not summing to 1.""" try: AutonomyController( weights={ "past_success_rate": 0.3, "codebase_familiarity": 0.3, "risk_assessment": 0.3, "invariant_complexity": 0.3, } ) context.error = None except ValueError as exc: context.error = exc