hurui200320
45941de6b1
Remove the loop for monitoring the CPU usage, adopt changes in the management service for the reporting message format, update the timing for sending out report, similar to java adapter. ISSUES RELATED: clevermicro/amq-adapter-python#58
225 lines
8.4 KiB
Python
225 lines
8.4 KiB
Python
import asyncio
|
|
import json
|
|
import time
|
|
from asyncio import AbstractEventLoop
|
|
from threading import Thread
|
|
|
|
from aio_pika import Message
|
|
from aio_pika.abc import AbstractRobustChannel
|
|
|
|
from amqp.adapter.logging_utils import logging_info
|
|
from amqp.config.amq_configuration import AMQConfiguration
|
|
from amqp.router.utils import await_future, await_result
|
|
|
|
CPU_IDLE, CPU_ACTIVE, CPU_OVERLOAD = 0, 1, 2
|
|
IDLE_THRESHOLD, OVERLOAD_THRESHOLD = 0.15, 0.90
|
|
RESOURCE_UNSET, RESOURCE_IDLE, RESOURCE_ACTIVE, RESOURCE_OVERLOAD = -1, 0, 1, 2
|
|
|
|
|
|
class ScaleRequestV1:
|
|
|
|
def __init__(
|
|
self,
|
|
serviceId: str,
|
|
taskId: str,
|
|
currentAvailability: int,
|
|
):
|
|
self.version = 1 # Version of the request, currently 1
|
|
self.serviceId = serviceId
|
|
self.taskId = taskId
|
|
self.currentAvailability = currentAvailability
|
|
self.thread = None
|
|
|
|
def to_json(self) -> str:
|
|
"""Converts the object to a JSON string matching the Java structure"""
|
|
return json.dumps(
|
|
{
|
|
"version": self.version,
|
|
"serviceId": self.serviceId,
|
|
"taskId": self.taskId,
|
|
"currentAvailability": self.currentAvailability,
|
|
},
|
|
indent=2,
|
|
)
|
|
|
|
|
|
class RunningAverage:
|
|
"""
|
|
A class to maintain a running average of the last N values.
|
|
The intended use is to track CPU usage over time window, that is directly related to sample rate and window size.
|
|
This is the reason why the constructor takes the number of items to store, calculated as window size / sample rate
|
|
Initial_value is for unit test to emulate OVERLOAD or IDLE condition.
|
|
"""
|
|
|
|
def __init__(self, num_items, initial_value=0):
|
|
self.buffer = [initial_value] * num_items # Fixed-size array
|
|
self.pointer = 0 # Current position in array
|
|
self.num_items = num_items # Maximum number of items to store
|
|
self.is_filled = False # Track if buffer is fully filled
|
|
|
|
def add(self, value):
|
|
# Store the value at current pointer position
|
|
self.buffer[self.pointer] = value
|
|
|
|
# Move pointer to next position with wrap-around
|
|
self.pointer += 1
|
|
if self.pointer >= self.num_items:
|
|
self.pointer = 0
|
|
self.is_filled = True
|
|
|
|
def get_average(self):
|
|
# Determine how many items we should average
|
|
count = self.num_items if self.is_filled else self.pointer
|
|
|
|
if count == 0:
|
|
return 0.0 # Avoid division by zero
|
|
|
|
return sum(self.buffer) / count
|
|
|
|
def get_current_values(self):
|
|
"""Returns the values in chronological order (oldest first)"""
|
|
if not self.is_filled:
|
|
return self.buffer[: self.pointer]
|
|
|
|
return self.buffer[self.pointer :] + self.buffer[: self.pointer]
|
|
|
|
|
|
class BackpressureHandler:
|
|
# Track the number of messages currently being processed
|
|
current_parallel_executions = 0
|
|
# report when remain availability goes down
|
|
last_usage_report_time = 0
|
|
# report when remain availability goes up
|
|
last_recover_report_time = 0
|
|
# _callback_list is used in unit tests to record the invoked callbacks
|
|
_callback_list = None
|
|
|
|
def __init__(
|
|
self,
|
|
channel: AbstractRobustChannel,
|
|
loop: AbstractEventLoop,
|
|
config: AMQConfiguration,
|
|
):
|
|
self.lock = None
|
|
self.channel = channel
|
|
self.loop = loop
|
|
self.config = config
|
|
self.exchange = None
|
|
self.swarm_service_id = self.config.amq_adapter.swarm_service_id
|
|
self.swarm_task_id = self.config.amq_adapter.swarm_task_id
|
|
self.parallel_workers = self.config.backpressure.threshold_threads
|
|
self.do_loop = -1
|
|
|
|
def _do_loop(self) -> bool:
|
|
"""
|
|
Check if the loop should continue running.
|
|
Positive value of do_loop indicates the number of iterations left.
|
|
Negative value indicates the loop should run indefinitely.
|
|
"""
|
|
_val = self.do_loop != 0
|
|
if self.do_loop > 0:
|
|
self.do_loop -= 1
|
|
return _val
|
|
|
|
async def increase_parallel_executions(self):
|
|
"""Increase the number of parallel executions"""
|
|
self.current_parallel_executions += 1
|
|
logging_info(
|
|
"Backpressure: Increased parallel executions, new value: %s",
|
|
self.current_parallel_executions,
|
|
)
|
|
# increase parallel -> reduce in remaining availability -> usage
|
|
# report usage at least every 1s
|
|
# TODO: add config for the interval
|
|
if self.last_usage_report_time + 1 < time.time():
|
|
await self.send_backpressure_report()
|
|
self.last_usage_report_time = time.time()
|
|
|
|
async def decrease_parallel_executions(self):
|
|
"""Decrease the number of parallel executions"""
|
|
if self.current_parallel_executions > 0:
|
|
self.current_parallel_executions -= 1
|
|
logging_info(
|
|
"Backpressure: Decreased parallel executions, new value=%s",
|
|
self.current_parallel_executions,
|
|
)
|
|
# decrease parallel -> add to remaining availability -> recover
|
|
# report recover at every 3s
|
|
# TODO: add config for the interval
|
|
if self.last_recover_report_time + 3 < time.time():
|
|
await self.send_backpressure_report()
|
|
self.last_recover_report_time = time.time()
|
|
|
|
def start_backpressure_monitor(self) -> Thread:
|
|
# Start the Backpressure monitor loop
|
|
self.thread = Thread(target=self.regular_report_loop_wrapper)
|
|
self.thread.daemon = True # This makes it a daemon thread
|
|
self.thread.start()
|
|
return self.thread
|
|
|
|
async def _regular_report_loop(self):
|
|
"""Regularly report the availability in a fixed interval"""
|
|
while self._do_loop():
|
|
await self.send_backpressure_report()
|
|
# report recover at every 5s
|
|
# TODO: add config for the interval
|
|
await asyncio.sleep(5)
|
|
|
|
def regular_report_loop_wrapper(self):
|
|
_loop = asyncio.new_event_loop()
|
|
_loop.run_until_complete(self._regular_report_loop())
|
|
|
|
async def send_backpressure_report(self):
|
|
scaling_request: ScaleRequestV1 = ScaleRequestV1(
|
|
serviceId=self.swarm_service_id,
|
|
taskId=self.swarm_task_id,
|
|
currentAvailability=self.parallel_workers - self.current_parallel_executions,
|
|
)
|
|
await self.publish_backpressure_request(scaling_request)
|
|
|
|
async def publish_backpressure_request(self, scaling_request: ScaleRequestV1):
|
|
# Publish the backpressure request to the management service
|
|
logging_info(
|
|
f"Publishing backpressure for {scaling_request.serviceId} with availability {scaling_request.currentAvailability}"
|
|
)
|
|
if not self.exchange:
|
|
|
|
async def _wrap_rabbit_mq_api_init(channel):
|
|
_exchange = await channel.get_exchange(name="cleverthis.clevermicro.management")
|
|
return _exchange
|
|
|
|
if BackpressureHandler._callback_list is not None:
|
|
BackpressureHandler._callback_list["_wrap_rabbit_mq_api_init"] = 1
|
|
self.exchange = await await_result(
|
|
asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api_init(self.channel), self.loop)
|
|
)
|
|
|
|
if self.exchange:
|
|
|
|
async def _wrap_rabbit_mq_api():
|
|
if not self.channel.is_closed:
|
|
binary_content: bytes = scaling_request.to_json().encode("utf-8")
|
|
pika_message: Message = Message(
|
|
body=binary_content,
|
|
content_encoding="utf-8",
|
|
delivery_mode=2,
|
|
content_type="application/octet-stream",
|
|
headers=None,
|
|
priority=0,
|
|
correlation_id=None,
|
|
)
|
|
await self.exchange.publish(
|
|
message=pika_message, routing_key="backpressure-scaling-v1"
|
|
)
|
|
logging_info(
|
|
"Service Message Published to %s, msg: %s",
|
|
self.exchange.name,
|
|
str(binary_content),
|
|
)
|
|
return True
|
|
return False
|
|
|
|
if BackpressureHandler._callback_list is not None:
|
|
BackpressureHandler._callback_list["_wrap_rabbit_mq_api"] = 1
|
|
await await_future(asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api(), self.loop))
|