clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Add support for file streaming upload, update to async pika lib

Browse Source
This commit is contained in:
Stanislav Hejny
2025-03-26 13:54:30 +00:00
parent e8f4abdfaf
commit a2ddce1a2e
22 changed files with 1369 additions and 395 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/router/router_producer.py
+52 -61
View File
@@ -7,10 +7,10 @@
* forwards it to the application.
"""
import logging
import os
import pika
from pika import BasicProperties, spec
from pika.channel import Channel
from aio_pika import Message
from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractRobustConnection, AbstractRobustExchange, \
AbstractIncomingMessage
from pika.exchange_type import ExchangeType
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
@@ -29,26 +29,18 @@ class RouterProducer(RouterBase):
* :param configuration: adapter's configuration reference.
"""
super().__init__()
self.connection = None
self.connection: AbstractRobustConnection | None = None
self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
self.amq_configuration: AMQConfiguration = configuration
self.channel: Channel | None = None
self.channel: AbstractRobustChannel | None = None
self.cm_request_exchange: AbstractRobustExchange | None = None
self.cm_response_exchange: AbstractRobustExchange | None = None
logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
self.amq_configuration.amq_adapter.service_name,
self.amq_configuration.amq_adapter.route_mapping)
def amq_consumer(self):
credentials = pika.PlainCredentials(
os.environ['RABBIT_MQ_USER'], os.environ['RABBIT_MQ_PASSWORD']
)
self.connection: pika.SelectConnection = pika.SelectConnection(
pika.ConnectionParameters(host=self.amq_configuration.dispatch.amq_host, credentials=credentials)
)
self.channel = self.connection.channel()
# self.channel.basic_consume()
def init_routing_paths(self, channel: Channel | None):
async def init_routing_paths(self, channel: AbstractRobustChannel | None):
"""
* Initialize router according to Producer mode.
* The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
@@ -63,22 +55,25 @@ class RouterProducer(RouterBase):
try:
# **** set up the service discovery internal exchanges ****
# 1. declare RequestExchange to where to publish the RoutingData requests
self.channel.exchange_declare(exchange=CM_REQUEST_EXCHANGE, exchange_type=ExchangeType.fanout)
self.cm_request_exchange = await self.channel.declare_exchange(
name=CM_REQUEST_EXCHANGE, type=ExchangeType.fanout
)
#
# 2. declare ResponseExchange where to publish current RoutingData (response to
# a RoutingData request)
self.channel.exchange_declare(exchange=CM_RESPONSE_EXCHANGE, exchange_type=ExchangeType.fanout)
self.cm_response_exchange = await self.channel.declare_exchange(
name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout
)
# 2a. declare a Response queue for Routing Data replies from Services
cm_response_queue: str = self.get_response_queue_name()
self.channel.queue_declare(cm_response_queue, durable=True, exclusive=False, auto_delete=False,
arguments={})
response_queue: AbstractRobustQueue = await self.channel.declare_queue(
name=cm_response_queue, durable=True, exclusive=False, auto_delete=False,
arguments={}
)
# 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
self.channel.queue_bind(queue=cm_response_queue, exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
await response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
# 2c. listen for incoming RoutingData messages
self.channel.basic_consume(
queue=cm_response_queue,
auto_ack=False,
on_message_callback=self.handle_routing_data_message)
await response_queue.consume(callback=self.handle_routing_data_message, no_ack=False)
logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
except Exception as ioe:
logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
@@ -86,38 +81,34 @@ class RouterProducer(RouterBase):
else:
logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
def handle_routing_data_message(self,
cb_channel: Channel,
method: spec.Basic.Deliver,
properties: spec.BasicProperties,
body: bytes):
async def handle_routing_data_message(self, message: AbstractIncomingMessage):
"""
* Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
* routing data, invokes addRoute or removeRoute depending on the type of the message.
"""
logging.info(f" [x] Received {body}, m={method}, p={properties}")
logging.info(f" [x] Received {message.body}, m={message.message_id}, p={message.properties}")
"""
logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
delivery.getEnvelope().getDeliveryTag(),
delivery.getEnvelope().toString(), str(delivery.getBody()))
"""
# ACK message now so that it is not being redelivered
cb_channel.basic_ack(method.delivery_tag, False)
message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
await message.ack(multiple=False)
cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
# ignore the message if it comes from ourselves (even if from different instance)
logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
method.delivery_tag,
self.service_message_factory.to_string(message),
message.delivery_tag,
self.service_message_factory.to_string(cm_message),
self.amq_configuration.amq_adapter.route_mapping,
self.amq_configuration.amq_adapter.service_name,
message.reply_to)
if message.is_valid() and not self.amq_configuration.amq_adapter.service_name == message.reply_to:
if message.message_type == ServiceMessageType.ROUTING_DATA:
self.add_routes(message.message)
if message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
self.remove_routes(message.message)
cm_message.reply_to)
if cm_message.is_valid() and not self.amq_configuration.amq_adapter.service_name == cm_message.reply_to:
if cm_message.message_type == ServiceMessageType.ROUTING_DATA:
self.add_routes(cm_message.message)
if cm_message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
self.remove_routes(cm_message.message)
def request_routes_from_adapters(self):
async def request_routes_from_adapters(self):
"""
* Send a broadcast message prompting all listening adapters to reply with own routing data.
"""
@@ -126,37 +117,37 @@ class RouterProducer(RouterBase):
serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
)
if not self.publish_service_message(serviceMessage, CM_REQUEST_EXCHANGE):
if not await self.publish_service_message(serviceMessage, self.cm_request_exchange):
logging.warning(" [E] RouterProducer could not request route data: %s channel is not open.",
CM_REQUEST_EXCHANGE)
except Exception as ioe:
logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
BASIC: BasicProperties = BasicProperties(content_type="application/octet-stream",
content_encoding=None,
headers=None,
delivery_mode=1,
priority=0,
correlation_id=None)
def publish_service_message(self, message: CleverMicroServiceMessage, exchange_name: str) -> bool:
async def publish_service_message(self,
message: CleverMicroServiceMessage,
exchange: AbstractRobustExchange) -> bool:
"""
* Publishes a service message to the service queue and logs success log entry.
*
* :param message: Service Message to be sent
* :param exchangeName: target exchange name where to publish the message
* :param queueName target: queue name (Optional, purpose is to get the queue size only)
* :param exchange: target exchange name where to publish the message
* :return True if channel is open, False otherwise
* @throws IOException if something else goes wrong
"""
if self.is_open(self.channel):
binary_content: bytes = self.service_message_factory.serialize(message)
self.channel.basic_publish(exchange=exchange_name,
routing_key="", # empty routing key
properties=self.BASIC,
body=binary_content)
logging.info(" [x] Service Message Sent to %s, msg: %s", exchange_name, str(binary_content))
pika_message: Message = Message(
body=binary_content,
content_encoding='utf-8',
delivery_mode=2,
content_type="application/octet-stream",
headers=None,
priority=0,
correlation_id=None
)
await exchange.publish(message=pika_message, routing_key="")
logging.info(" [x] Service Message Sent to %s, msg: %s", exchange.name, str(binary_content))
return True
return False
@@ -181,10 +172,10 @@ class RouterProducer(RouterBase):
"""
return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
def is_open(self, _channel: Channel) -> bool:
def is_open(self, _channel: AbstractRobustChannel) -> bool:
"""
* Test if the channel is usable (opened).
* :param _channel: the channel
* :return True if the channel is opened.
"""
return _channel is not None and _channel.is_open
return _channel is not None and not _channel.is_closed