feat/58 - simplify complex logic

amqp/adapter/backpressure_handler.py

@@ -5,7 +5,6 @@ from asyncio import AbstractEventLoop
 from threading import Thread
 from typing import Optional
 
-import psutil
 from aio_pika import Message
 from aio_pika.abc import AbstractRobustChannel
 
@@ -14,10 +13,6 @@ from amqp.config.amq_configuration import AMQConfiguration
 from amqp.model.model import ScalingRequestAlert
 from amqp.router.utils import await_future, await_result
 
-CPU_IDLE, CPU_ACTIVE, CPU_OVERLOAD = 0, 1, 2
-IDLE_THRESHOLD, OVERLOAD_THRESHOLD = 0.15, 0.90
-RESOURCE_UNSET, RESOURCE_IDLE, RESOURCE_ACTIVE, RESOURCE_OVERLOAD = -1, 0, 1, 2
-
 
 class ScaleRequestV1:
 
@@ -25,15 +20,15 @@ class ScaleRequestV1:
         self,
         serviceId: str,
         taskId: str,
-        maxAvailability: int,
-        currentAvailability: int,
+        max_availability: int,
+        current_availability: int,
         requestType: ScalingRequestAlert,
     ):
         self.version = 1  # Version of the request, currently 1
         self.serviceId = serviceId
         self.taskId = taskId
-        self.maxAvailability = maxAvailability
-        self.currentAvailability = currentAvailability
+        self.max_availability = max_availability
+        self.current_availability = current_availability
         self.requestType = requestType
         self.thread = None
 
@@ -44,58 +39,17 @@ class ScaleRequestV1:
                 "version": self.version,
                 "serviceId": self.serviceId,
                 "taskId": self.taskId,
-                "maxAvailability": self.maxAvailability,
-                "currentAvailability": self.currentAvailability,
+                "maxAvailability": self.max_availability,
+                "currentAvailability": self.current_availability,
                 "requestType": self.requestType.value,  # Using .value for Enum serialization
             },
             indent=2,
         )
 
 
-class RunningAverage:
-    """
-    A class to maintain a running average of the last N values.
-    The intended use is to track CPU usage over time window, that is directly related to sample rate and window size.
-    This is the reason why the constructor takes the number of items to store, calculated as window size / sample rate
-    Initial_value is for unit test to emulate OVERLOAD or IDLE condition.
-    """
-
-    def __init__(self, num_items, initial_value=0):
-        self.buffer = [initial_value] * num_items  # Fixed-size array
-        self.pointer = 0  # Current position in array
-        self.num_items = num_items  # Maximum number of items to store
-        self.is_filled = False  # Track if buffer is fully filled
-
-    def add(self, value):
-        # Store the value at current pointer position
-        self.buffer[self.pointer] = value
-
-        # Move pointer to next position with wrap-around
-        self.pointer += 1
-        if self.pointer >= self.num_items:
-            self.pointer = 0
-            self.is_filled = True
-
-    def get_average(self):
-        # Determine how many items we should average
-        count = self.num_items if self.is_filled else self.pointer
-
-        if count == 0:
-            return 0.0  # Avoid division by zero
-
-        return sum(self.buffer) / count
-
-    def get_current_values(self):
-        """Returns the values in chronological order (oldest first)"""
-        if not self.is_filled:
-            return self.buffer[: self.pointer]
-
-        return self.buffer[self.pointer :] + self.buffer[: self.pointer]
-
-
 class BackpressureHandler:
     # Track the number of messages currently being processed
-    current_load = 0
+    current_availability = 0
     # helps detect IDLE condition
     # helps prevent flooding the system with backpressure events
     last_backpressure_event_time = 0
@@ -116,17 +70,19 @@ class BackpressureHandler:
         self.exchange = None
         self.swarm_service_id = self.config.amq_adapter.swarm_service_id
         self.swarm_task_id = self.config.amq_adapter.swarm_task_id
-        self.maximum_load = self.config.backpressure.threshold_load
-        self.average_cpu_usage = None
         self.do_loop = -1
         self._resource_usage_changed = 0
         self._resource_average_value = 0
         self._last_resource_max_value = 100  # Default max value for CPU usage
+        self.max_availability = -1
+        self.current_availability = 1
+        self.max_load = 1
+        self.current_load = 0
 
     def increase_current_load(self):
-        """Increase the number of parallel executions, or current load"""
+        """Increase the number of parallel executions, or current_availability load"""
         logging_info(
-            "Backpressure: Increase parallel executions, current=%s",
+            "Backpressure: Increase current load (%s) by 1",
             self.current_load,
         )
         self.current_load += 1
@@ -134,55 +90,43 @@ class BackpressureHandler:
     def decrease_current_load(self):
         """Decrease the number of parallel executions, or current load"""
         logging_info(
-            "Backpressure: Decrease parallel executions, current=%s",
+            "Backpressure: Decrease current load(%s) by 1",
             self.current_load,
         )
         if self.current_load > 0:
             self.current_load -= 1
 
-    def update_current_load(self, count: int):
+    def update_current_availability(self, count: int):
         """Update the number of parallel executions, or current load"""
-        self.current_load = count
+        self.current_availability = count
 
-    def update_last_data_message_time(self):
+    def update_last_backpressure_event_time(self):
         # logging_info("Backpressure: Update last data message time")
         """Update the last data message time"""
         self.last_backpressure_event_time = time.time()
 
-    async def update_backpressure_value(self, current: int, maximum: int):
+    async def update_backpressure_value(self, current_availability: int, maximum: int):
         """
         Update the current backpressure value and check for overload conditions.
         This method is called by the AMQService.backpressure() method to update
         the current parallel executions count and check for overload conditions.
 
         Args:
-            current: Current value of the backpressure metric
+            current_availability: Current value of the backpressure metric
             maximum: Maximum value of the backpressure metric
         """
         logging_info(
-            "Backpressure: Updating backpressure value, current=%s, maximum=%s", current, maximum
+            "Backpressure: Updating backpressure value, current_availability=%s, maximum=%s",
+            current_availability,
+            maximum,
         )
-        if maximum > 0 and maximum > self.maximum_load:
-            self.maximum_load = maximum
-
-        # Update the current parallel executions count
-        self.update_current_load(current)
-
+        if maximum > 0 and maximum > self.max_availability:
+            self.max_availability = maximum
+        # Update the current_availability / parallel executions count
+        self.update_current_availability(current_availability)
         # Check for overload conditions
         await self.check_overload_condition()
 
-        # Update the last data message time to indicate activity
-        self.update_last_data_message_time()
-
-        # If maximum has changed, update the parallel workers threshold
-        if maximum != self.maximum_load and maximum > 0:
-            logging_info(
-                "Backpressure: Updating maximum parallel workers from %s to %s",
-                self.maximum_load,
-                maximum,
-            )
-            self.maximum_load = maximum
-
     def start_backpressure_monitor(self) -> Optional[Thread]:
         # Start the Backpressure monitor loop
         self.thread = Thread(target=self.backpressure_monitor_loop)
@@ -192,42 +136,38 @@ class BackpressureHandler:
 
     async def check_overload_condition(self):
         """
-        Check if the current availability is too low (OVERLOAD)
+        Check if the current_availability availability is too low (OVERLOAD)
         or if the service has been idle for too long with high availability (IDLE).
-        
+
         Note: current_availability represents available capacity, not used capacity.
         - Low availability (close to 0) means OVERLOAD
         - High availability (close to maximum) with no activity means IDLE
         """
         current_time = time.time()
-        last_event_delta = current_time - self.last_backpressure_event_time
-        
-        # If maximum is not set or invalid, use the highest seen value
-        if self.maximum_load <= 0 and self.current_load > 0:
-            self.maximum_load = self.current_load
-        elif self.current_load > self.maximum_load:
-            # Update maximum if we see a higher value
-            self.maximum_load = self.current_load
-        
+        last_event_delta: float = current_time - self.last_backpressure_event_time
+
         logging_info(
             "Backpressure: Check conditions, current_availability=%s, max=%s, last_activity=%s",
-            self.current_load,
-            self.maximum_load,
+            self.current_availability,
+            self.max_availability,
             last_event_delta,
         )
-    
+
         # Check for OVERLOAD condition - low availability (less than 20% of maximum)
-        if self.current_load <= round(0.2 * self.maximum_load):
+        if self.current_availability <= round(0.1 * self.max_availability):
             # Check if the last backpressure event was not an overload
-            if self.last_backpressure_event != ScalingRequestAlert.OVERLOAD:
+            if (
+                self.last_backpressure_event != ScalingRequestAlert.OVERLOAD
+                or last_event_delta > self.config.backpressure.idle_duration
+            ):
                 # Trigger the overload event
                 await self.handle_backpressure_overload_event()
                 # update / reset time-window so that the OVERLOAD is not sent too often
                 self.last_backpressure_event_time = current_time
                 self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
-    
+
         # Check for IDLE condition - high availability (more than 80% of maximum) with no activity
-        elif self.current_load >= round(0.8 * self.maximum_load):
+        elif self.current_availability >= round(0.8 * self.max_availability):
             idle_duration = self.config.backpressure.idle_duration
             # Check if service has been idle for longer than the configured duration
             if (
@@ -239,118 +179,46 @@ class BackpressureHandler:
                     last_event_delta,
                     idle_duration,
                 )
-    
+
                 # Trigger the idle event
                 await self._handle_backpressure_idle_event()
                 # update / reset time-window so that the IDLE is not sent too often
                 self.last_backpressure_event_time = current_time
             else:
-                # Don't send IDLE right away when the availability increases, but wait for a while
+                # Don't send IDLE right away when the availability increases, but wait for a while, send UPDATE instead
                 self.last_backpressure_event = ScalingRequestAlert.IDLE
-    
+                _scaling_request: ScaleRequestV1 = ScaleRequestV1(
+                    self.swarm_service_id,
+                    self.swarm_task_id,
+                    self.max_availability,
+                    self.current_availability,  # Current availability is passed directly
+                    ScalingRequestAlert.UPDATE,
+                )
+                # Address the message to any adapter capable of supporting BACKPRESSURE request
+                await self.publish_backpressure_request(_scaling_request)
+                self.last_backpressure_event_time = current_time
+
         # If neither OVERLOAD nor IDLE, and it's time for an update, send UPDATE event
         elif last_event_delta > self.config.backpressure.time_window:
             _scaling_request: ScaleRequestV1 = ScaleRequestV1(
                 self.swarm_service_id,
                 self.swarm_task_id,
-                self.maximum_load,
-                self.current_load,  # Current availability is passed directly
+                self.max_availability,
+                self.current_availability,  # Current availability is passed directly
                 ScalingRequestAlert.UPDATE,
             )
             # Address the message to any adapter capable of supporting BACKPRESSURE request
             await self.publish_backpressure_request(_scaling_request)
             self.last_backpressure_event_time = current_time
             self.last_backpressure_event = ScalingRequestAlert.UPDATE
-    
-        self.update_last_data_message_time()
+
+        self.update_last_backpressure_event_time()
 
     async def _backpressure_monitor(self):
-        """Monitor the backpressure conditions and trigger events accordingly"""
-        # Check if CPU monitoring is enabled in configuration
-        cpu_monitoring_enabled = self.config.backpressure.cpu_monitoring_enabled
-        _time_window_millis = self.config.backpressure.time_window
-        _idle_duration_millis = self.config.backpressure.idle_duration
-        _overload_duration_millis = self.config.backpressure.cpu_overload_duration
+        """Periodically monitor the backpressure conditions and trigger events accordingly"""
         _monitor_interval = 0.5  # Monitor every 500ms
-        CPU_IDLE, CPU_ACTIVE, CPU_OVERLOAD = 0, 1, 2
-        IDLE_THRESHOLD, OVERLOAD_THRESHOLD = 15, 90
-        if self.average_cpu_usage is None:
-            self.average_cpu_usage = RunningAverage(
-                int(max(_overload_duration_millis, _idle_duration_millis) // _monitor_interval)
-            )
-        _cpu_usage_state = CPU_ACTIVE
-        _cpu_usage_changed = time.time()
         while self._do_loop():
-            if cpu_monitoring_enabled:
-                _current_cpu_usage = psutil.cpu_percent(interval=None)
-                self.average_cpu_usage.add(_current_cpu_usage)
-                _average_cpu_usage = self.average_cpu_usage.get_average()
-                _now = time.time()
-                # logging_info(
-                #    "Backpressure: current CPU[pct]=%s, avg=%s, state=%s, last dataMsg=%s, last event=%s, last eventTime=%s",
-                #    _current_cpu_usage,
-                #    _average_cpu_usage,
-                #    _cpu_usage_state,
-                #    self.last_backpressure_event,
-                #    self.last_backpressure_event_time,
-                # )
-                if _average_cpu_usage < IDLE_THRESHOLD:
-                    if _cpu_usage_state != CPU_IDLE:
-                        _cpu_usage_changed = _now
-                    _cpu_usage_state = CPU_IDLE
-                elif _average_cpu_usage > OVERLOAD_THRESHOLD:
-                    if _cpu_usage_state != CPU_OVERLOAD:
-                        _cpu_usage_changed = _now
-                    _cpu_usage_state = CPU_OVERLOAD
-                else:
-                    if _cpu_usage_state != CPU_ACTIVE:
-                        _cpu_usage_changed = _now
-                    _cpu_usage_state = CPU_ACTIVE
-
-                # Check if the current time exceeds the last backpressure event time
-                if (
-                    _cpu_usage_state == CPU_OVERLOAD
-                    and _now - _cpu_usage_changed > _overload_duration_millis
-                    and (
-                        _now - self.last_backpressure_event_time > _time_window_millis
-                        or self.last_backpressure_event != ScalingRequestAlert.OVERLOAD
-                    )
-                ):
-                    # Trigger the overload event
-                    await self.handle_backpressure_overload_event()
-                    self.last_backpressure_event_time = _now
-                    self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
-                elif (
-                    _cpu_usage_state == CPU_IDLE
-                    and _now - _cpu_usage_changed > _idle_duration_millis
-                    and (
-                        _now - self.last_backpressure_event_time > _time_window_millis
-                        or self.last_backpressure_event != ScalingRequestAlert.IDLE
-                    )
-                ):
-                    # Trigger the idle event
-                    await self._handle_backpressure_idle_event()
-                    self.last_backpressure_event = ScalingRequestAlert.IDLE
-                    self.last_backpressure_event_time = _now
-                else:
-                    # Trigger update event
-                    if _now - self.last_backpressure_event_time > _time_window_millis:
-                        # Trigger the update event
-                        _scaling_request: ScaleRequestV1 = ScaleRequestV1(
-                            self.swarm_service_id,
-                            self.swarm_task_id,
-                            100,
-                            _average_cpu_usage,
-                            ScalingRequestAlert.UPDATE,
-                        )
-                        # Address the message to any adapter capable of supporting BACKPRESSURE request
-                        await self.publish_backpressure_request(_scaling_request)
-                        self.last_backpressure_event_time = _now
-                        self.last_backpressure_event = ScalingRequestAlert.UPDATE
-            else:
-                # If CPU monitoring is disabled, just check the current load
-                await self.check_overload_condition()
-
+            await self.check_overload_condition()
             await asyncio.sleep(_monitor_interval)
 
     def backpressure_monitor_loop(self):
@@ -363,8 +231,8 @@ class BackpressureHandler:
         scaling_request: ScaleRequestV1 = ScaleRequestV1(
             self.swarm_service_id,
             self.swarm_task_id,
-            self.maximum_load,
-            self.current_load,  # Current availability is passed directly
+            self.max_availability,
+            self.current_availability,  # Current availability is passed directly
             ScalingRequestAlert.OVERLOAD,
         )
         # Address the message to any management service instance capable of supporting BACKPRESSURE request
@@ -376,8 +244,8 @@ class BackpressureHandler:
         scaling_request: ScaleRequestV1 = ScaleRequestV1(
             self.swarm_service_id,
             self.swarm_task_id,
-            self.maximum_load,
-            self.current_load,  # Current availability is passed directly
+            self.max_availability,
+            self.current_availability,  # Current availability is passed directly
             ScalingRequestAlert.IDLE,
         )
         # Address the message to any adapter capable of supporting BACKPRESSURE request

amqp/adapter/cleverthis_service_adapter.py

@@ -83,7 +83,7 @@ async def _convert_file_response(
     temporary dedicated queue.
     """
     # All calls to RabbitMQ API have to be called threadsafe, because those appear to be attached
-    # to parent event loop, and need to execute at that event loop, not the current one
+    # to parent event loop, and need to execute at that event loop, not the current_availability one
     # allow coroutine to execute. calling _future.result() will block the entire thread
     # and the coroutine will never execute
     _channel, _exchange, _routing_key = await await_result(
@@ -241,7 +241,7 @@ class CleverThisServiceAdapter:
 
     async def get_current_user(self, token: str) -> object:
         """
-        To Be Overriden in subclass for given CleverThis Service, to return the current user based
+        To Be Overriden in subclass for given CleverThis Service, to return the current_availability user based
         on the token, in format appropriate for the service.
 
         :param token: The token to be used for authentication.

amqp/adapter/logging_utils.py

@@ -13,17 +13,17 @@ __verbose__ = os.environ.get("AMQ_VERBOSE_LOGGING", "0") == "1"
 
 def get_context_info():
     """
-    Retrieves the current module, line number, and function name.
+    Retrieves the current_availability module, line number, and function name.
 
     Returns:
         A tuple containing (module name, line number, function name).
         Returns (None, None, None) if it fails to retrieve the information.
     """
     try:
-        # Get the frame object for the current function call
+        # Get the frame object for the current_availability function call
         frame = inspect.currentframe()
         if frame is not None:
-            # Get the frame object for the caller of the current function
+            # Get the frame object for the caller of the current_availability function
             frame = frame.f_back.f_back
         if frame:
             code_context = frame.f_code
@@ -44,7 +44,7 @@ def get_context_info():
 
 def logging_error(msg: str, *args) -> None:
     """
-    Log an error message according to current logging for 'ERROR' level.
+    Log an error message according to current_availability logging for 'ERROR' level.
     Add module name, line and function name where the error occurred, on single line.
     Args:
         msg (str): The error message to log.
@@ -70,7 +70,7 @@ def logging_error(msg: str, *args) -> None:
 
 def logging_warning(msg: str, *args) -> None:
     """
-    Log a warning message according to current logging for 'WARN' level.
+    Log a warning message according to current_availability logging for 'WARN' level.
     Add module name, line and function name where the print occurred, on single line.
     Args:
         msg (str): The warning message to log.
@@ -91,7 +91,7 @@ def logging_warning(msg: str, *args) -> None:
 
 def logging_info(msg: str, *args) -> None:
     """
-    Log an info message according to current logging for 'INFO' level.
+    Log an info message according to current_availability logging for 'INFO' level.
     Add module name, line and function name where the print occurred, on single line.
     Args:
         msg (str): The info message to log.
@@ -112,7 +112,7 @@ def logging_info(msg: str, *args) -> None:
 
 def logging_debug(msg: str, *args) -> None:
     """
-    Log a debug message according to current logging for 'DEBUG' level.
+    Log a debug message according to current_availability logging for 'DEBUG' level.
     Add module name, line and function name where the print occurred, on single line.
     Args:
         msg (str): The debug message to log.

amqp/config/amq_configuration.py

@@ -203,7 +203,7 @@ class AMQConfiguration:
             config.read(config_file_name)
         else:
             # Get the directory of the caller (the module where this function is called)
-            caller_frame = inspect.stack()[1]  # 0 is current frame, 1 is caller
+            caller_frame = inspect.stack()[1]  # 0 is current_availability frame, 1 is caller
             caller_module = inspect.getmodule(caller_frame[0])
             if caller_module:
                 caller_dir = os.path.dirname(os.path.abspath(caller_module.__file__))

amqp/router/router_consumer.py

@@ -174,12 +174,12 @@ class RouterConsumer(RouterProducer):
             )
             if not await self.publish_service_message(service_message, self.cm_response_exchange):
                 logging_warning(
-                    "RouterConsumer couldn't remove current route: %s, channel not open.",
+                    "RouterConsumer couldn't remove current_availability route: %s, channel not open.",
                     route,
                 )
 
         except Exception as ioe:
-            logging_error("RouterConsumer failed remove current route: %s", ioe)
+            logging_error("RouterConsumer failed remove current_availability route: %s", ioe)
 
         result: bool = self.remove_route(route)
         return result

amqp/router/router_producer.py

@@ -81,7 +81,7 @@ class RouterProducer(RouterBase):
                     name=CM_REQUEST_EXCHANGE, type=ExchangeType.fanout
                 )
                 #
-                # 2. declare ResponseExchange where to publish current RoutingData (response to
+                # 2. declare ResponseExchange where to publish current_availability RoutingData (response to
                 #    a RoutingData request)
                 self.cm_response_exchange = await self.channel.declare_exchange(
                     name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout

amqp/service/amq_message_handler.py

@@ -127,7 +127,7 @@ class DataMessageHandler:
         self.backpressure_handler.increase_current_load()
         await self.backpressure_handler.check_overload_condition()
         logging_info(
-            f"PARALLEL: Current Capacity [{self.backpressure_handler.current_load}] of [{self.backpressure_handler.maximum_load}]"
+            f"PARALLEL: Current Capacity [{self.backpressure_handler.current_availability}] of [{self.backpressure_handler.max_availability}]"
         )
         if message.reply_to:
             self.reply_to = message.reply_to
@@ -156,7 +156,6 @@ class DataMessageHandler:
             logging_error(f"Request handler: {e}")
 
         self.backpressure_handler.decrease_current_load()
-        await self.backpressure_handler.check_overload_condition()
 
     async def reconstitute_data_message(
         self, message: AbstractIncomingMessage

amqp/service/amq_service.py

@@ -64,6 +64,7 @@ class AMQService:
         )
         self.backpressure_thread: Optional[Thread] = None
         self.backpressure_handler: Optional[BackpressureHandler] = None
+        self.maximum_availability = -1
 
     # =======================================================================
     # =======================================================================
@@ -87,21 +88,28 @@ class AMQService:
         """
         Send Backpressure event to the Management Service. The Backpressure logic determines the type of the event
         based on the current_availability and maximum values.
-        - OVERLOAD event is triggered immediately when the current_availability value exceeds the maximum threshold, which is
-        typically set to 80% of the maximum value.
-        - IDLE event is triggered when the current_availability value drops below 20% of the maximum value and remains there for
-        a IDLE_WINDOW period of time (see BackpressureHandler).
+        - OVERLOAD event is triggered immediately when the current_availability value is below threshold
+        typically set to 10% of the maximum value, or 0 if no maximum is provided.
+        - IDLE event is triggered when the current_availability value exeeds 80% of the maximum value or is greater
+        than 0 if no maximum is provided.
         - UPDATE event is sent if neither OVERLOAD nor IDLE conditions are met, indicating that the backpressure is
         within acceptable limits.
-        :param current_availability: Currently available caopacity.
+        :param current_availability: Currently available capacity.
         :param maximum: Maximum value of the backpressure metric. If undefined or set to lt 0 (e.g. -1),
-        the maximum is than hgdetermined from max value of current_availability seen over the time.
+        the maximum is than determined from max value of current_availability seen over the time.
         """
         if self.backpressure_handler is not None:
+            # watch maximum & current value to always have some valid maximum reference
             if maximum < 0:
-                maximum = self.amq_configuration.backpressure.threshold_load
+                if current_availability > self.maximum_availability:
+                    self.maximum_availability = current_availability
+            else:
+                if maximum > self.maximum_availability:
+                    self.maximum_availability = maximum
             asyncio.run_coroutine_threadsafe(
-                self.backpressure_handler.update_backpressure_value(current_availability, maximum),
+                self.backpressure_handler.update_backpressure_value(
+                    current_availability, maximum if maximum > 0 else self.maximum_availability
+                ),
                 loop=self.backpressure_handler.loop,
             )