From 826c859b321e9eb256fa5b7377acdbc99d352efc Mon Sep 17 00:00:00 2001 From: Stanislav Hejny Date: Fri, 11 Jul 2025 23:54:57 +0100 Subject: [PATCH] feat: add IDLE event detection to check_overload_condition function Co-authored-by: aider (openrouter/openai/o3-mini-high) --- amqp/adapter/backpressure_handler.py | 47 ++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/amqp/adapter/backpressure_handler.py b/amqp/adapter/backpressure_handler.py index 53c4eec..8033f54 100644 --- a/amqp/adapter/backpressure_handler.py +++ b/amqp/adapter/backpressure_handler.py @@ -210,20 +210,61 @@ class BackpressureHandler: return self.thread async def check_overload_condition(self): - """Check if the current parallel executions exceed the limit""" + """ + Check if the current parallel executions exceed the limit (OVERLOAD) + or if the service has been idle for too long (IDLE). + """ + current_time = time.time() self.update_last_data_message_time() + logging_info( - "Backpressure: Check overload condition, current=%s, max=%s", + "Backpressure: Check conditions, current=%s, max=%s, last_activity=%s", self.current_parallel_executions, self.parallel_workers, + current_time - self.last_data_message_time, ) + + # Check for OVERLOAD condition if self.current_parallel_executions >= self.parallel_workers - 1: # Check if the last backpressure event was not an overload if self.last_backpressure_event != ScalingRequestAlert.OVERLOAD: # Trigger the overload event await self.handle_backpressure_overload_event() - self.last_backpressure_event_time = time.time() + self.last_backpressure_event_time = current_time self.last_backpressure_event = ScalingRequestAlert.OVERLOAD + + # Check for IDLE condition + elif self.current_parallel_executions == 0: + idle_duration = self.config.backpressure.idle_duration + # Check if service has been idle for longer than the configured duration + if (current_time - self.last_data_message_time > idle_duration and + (current_time - self.last_backpressure_event_time > self.config.backpressure.time_window or + self.last_backpressure_event != ScalingRequestAlert.IDLE)): + + logging_info( + "Backpressure: Service has been idle for %s seconds (threshold: %s seconds)", + current_time - self.last_data_message_time, + idle_duration + ) + + # Trigger the idle event + await self._handle_backpressure_idle_event() + self.last_backpressure_event_time = current_time + self.last_backpressure_event = ScalingRequestAlert.IDLE + + # If neither OVERLOAD nor IDLE, and it's time for an update, send UPDATE event + elif (current_time - self.last_backpressure_event_time > self.config.backpressure.time_window): + _scaling_request: ScaleRequestV1 = ScaleRequestV1( + self.swarm_service_id, + self.swarm_task_id, + self.parallel_workers, + self.parallel_workers - self.current_parallel_executions, + ScalingRequestAlert.UPDATE, + ) + # Address the message to any adapter capable of supporting BACKPRESSURE request + await self.publish_backpressure_request(_scaling_request) + self.last_backpressure_event_time = current_time + self.last_backpressure_event = ScalingRequestAlert.UPDATE async def _backpressure_monitor(self): """Monitor the backpressure conditions and trigger events accordingly"""