Add support for file streaming upload, update to async pika lib
This commit is contained in:
Stanislav Hejny
@@ -1,9 +1,7 @@
|
||||
import logging
|
||||
from typing import Set
|
||||
|
||||
from pika import spec
|
||||
from pika.channel import Channel
|
||||
|
||||
from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractIncomingMessage
|
||||
from amqp.adapter.amq_route_factory import AMQRouteFactory
|
||||
from amqp.config.amq_configuration import AMQConfiguration
|
||||
from amqp.model.model import CleverMicroServiceMessage, AMQRoute
|
||||
@@ -30,25 +28,21 @@ class RouterConsumer(RouterProducer):
|
||||
self.amqRouteFactory = AMQRouteFactory()
|
||||
logging.info(" [*] RouterConsumer.<init>")
|
||||
|
||||
def init_routing_paths_consumer(self, _channel: Channel):
|
||||
async def init_routing_paths_consumer(self, _channel: AbstractRobustChannel):
|
||||
"""
|
||||
* Initialize router according to Consumer mode.
|
||||
"""
|
||||
|
||||
self.init_routing_paths_consumer(_channel)
|
||||
logging.info(" [*] RouterConsumer.initConsumerRouter, channel open: %s", self.is_open(_channel))
|
||||
if self.is_open(_channel):
|
||||
try:
|
||||
# 1. declare a Queue that gives the RoutingDAta requests
|
||||
cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
|
||||
self.channel.queue_declare(cm_request_queue, durable=True, exclusive=False, auto_delete=False)
|
||||
queue: AbstractRobustQueue = await self.channel.declare_queue(name=cm_request_queue, durable=True, exclusive=False, auto_delete=False)
|
||||
# 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
|
||||
self.channel.queue_bind(queue=cm_request_queue, exchange=CM_REQUEST_EXCHANGE, routing_key="")
|
||||
await queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
|
||||
# 1b. Service adapter mode - listens on Request queue and replies to Response queue
|
||||
logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue)
|
||||
self.channel.basic_consume(queue=cm_request_queue,
|
||||
auto_ack=False,
|
||||
on_message_callback=self.handle_routing_data_request_message)
|
||||
await queue.consume(callback=self.handle_routing_data_request_message, no_ack=False)
|
||||
except Exception as ioe:
|
||||
logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe)
|
||||
else:
|
||||
@@ -58,39 +52,35 @@ class RouterConsumer(RouterProducer):
|
||||
* ServiceMessageType.ROUTING_DATA_REQUEST handler listening on queue 'CleverMicroRequest'
|
||||
"""
|
||||
|
||||
def handle_routing_data_request_message(self,
|
||||
cb_channel: Channel,
|
||||
method: spec.Basic.Deliver,
|
||||
properties: spec.BasicProperties,
|
||||
body: bytes):
|
||||
async def handle_routing_data_request_message(self, message: AbstractIncomingMessage):
|
||||
# some debug statements, TODO - remove for prod release
|
||||
delivery_tag = method.delivery_tag
|
||||
debug = "Delivery.Properties = " + properties.__str__()
|
||||
logging.info(" [DBG] ******[%] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
|
||||
delivery_tag, method.consumer_tag, method.get_properties(), debug)
|
||||
delivery_tag = message.delivery_tag
|
||||
debug = "Delivery.Properties = " + message.properties.__str__()
|
||||
logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
|
||||
delivery_tag, message.consumer_tag, message.properties, debug)
|
||||
|
||||
cb_channel.basic_ack(delivery_tag, False)
|
||||
await message.ack(False)
|
||||
|
||||
message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
|
||||
cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
|
||||
route_mapping: str = self.wrap_own_routes()
|
||||
logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
|
||||
delivery_tag, self.service_message_factory.to_string(message), route_mapping)
|
||||
delivery_tag, self.service_message_factory.to_string(cm_message), route_mapping)
|
||||
# Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
|
||||
if message.is_valid() and message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
|
||||
if cm_message.is_valid() and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
|
||||
try:
|
||||
service_message: CleverMicroServiceMessage = self.service_message_factory.of(
|
||||
ServiceMessageType.ROUTING_DATA, route_mapping, message.reply_to
|
||||
ServiceMessageType.ROUTING_DATA, route_mapping, cm_message.reply_to
|
||||
)
|
||||
self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE)
|
||||
await self.publish_service_message(service_message, self.cm_response_exchange)
|
||||
except Exception as err:
|
||||
logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
|
||||
delivery_tag, CM_RESPONSE_EXCHANGE, err)
|
||||
|
||||
else:
|
||||
logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
|
||||
delivery_tag, str(body))
|
||||
delivery_tag, str(message.body))
|
||||
|
||||
def register_route(self, route: AMQRoute):
|
||||
async def register_route(self, route: AMQRoute):
|
||||
"""
|
||||
* Called in Consumer (Service Adapter) context. Registers the routing data for self Service
|
||||
* with the Master router by sending own routing data to Master via CleverMicroResponse queue.
|
||||
@@ -101,7 +91,7 @@ class RouterConsumer(RouterProducer):
|
||||
serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
|
||||
ServiceMessageType.ROUTING_DATA, route.as_string(), ANY_RECIPIENT
|
||||
)
|
||||
self.publish_service_message(serviceMessage, CM_RESPONSE_EXCHANGE)
|
||||
await self.publish_service_message(serviceMessage, self.cm_response_exchange)
|
||||
self.add_own_route(route)
|
||||
logging.info(" [DBG] RouterConsumer registered Route: %s", route)
|
||||
else:
|
||||
@@ -110,7 +100,7 @@ class RouterConsumer(RouterProducer):
|
||||
except Exception as ioe:
|
||||
logging.error(" [E] RouterConsumer failed register route data: %s", ioe)
|
||||
|
||||
def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
|
||||
async def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
|
||||
"""
|
||||
* Unregister / remove route from the list
|
||||
* :param route: route to remove
|
||||
@@ -123,7 +113,7 @@ class RouterConsumer(RouterProducer):
|
||||
route.as_string(),
|
||||
ANY_RECIPIENT
|
||||
)
|
||||
if not self.publish_service_message(service_message, CM_RESPONSE_EXCHANGE):
|
||||
if not await self.publish_service_message(service_message, self.cm_response_exchange):
|
||||
logging.warning(" [W] RouterConsumer couldn't remove current route: %s, channel not open.",
|
||||
route)
|
||||
|
||||
|
||||
@@ -7,10 +7,10 @@
|
||||
* forwards it to the application.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import pika
|
||||
from pika import BasicProperties, spec
|
||||
from pika.channel import Channel
|
||||
|
||||
from aio_pika import Message
|
||||
from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractRobustConnection, AbstractRobustExchange, \
|
||||
AbstractIncomingMessage
|
||||
from pika.exchange_type import ExchangeType
|
||||
|
||||
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
|
||||
@@ -29,26 +29,18 @@ class RouterProducer(RouterBase):
|
||||
* :param configuration: adapter's configuration reference.
|
||||
"""
|
||||
super().__init__()
|
||||
self.connection = None
|
||||
self.connection: AbstractRobustConnection | None = None
|
||||
self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
|
||||
self.amq_configuration: AMQConfiguration = configuration
|
||||
self.channel: Channel | None = None
|
||||
self.channel: AbstractRobustChannel | None = None
|
||||
self.cm_request_exchange: AbstractRobustExchange | None = None
|
||||
self.cm_response_exchange: AbstractRobustExchange | None = None
|
||||
|
||||
logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
|
||||
self.amq_configuration.amq_adapter.service_name,
|
||||
self.amq_configuration.amq_adapter.route_mapping)
|
||||
|
||||
def amq_consumer(self):
|
||||
credentials = pika.PlainCredentials(
|
||||
os.environ['RABBIT_MQ_USER'], os.environ['RABBIT_MQ_PASSWORD']
|
||||
)
|
||||
self.connection: pika.SelectConnection = pika.SelectConnection(
|
||||
pika.ConnectionParameters(host=self.amq_configuration.dispatch.amq_host, credentials=credentials)
|
||||
)
|
||||
self.channel = self.connection.channel()
|
||||
# self.channel.basic_consume()
|
||||
|
||||
def init_routing_paths(self, channel: Channel | None):
|
||||
async def init_routing_paths(self, channel: AbstractRobustChannel | None):
|
||||
"""
|
||||
* Initialize router according to Producer mode.
|
||||
* The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
|
||||
@@ -63,22 +55,25 @@ class RouterProducer(RouterBase):
|
||||
try:
|
||||
# **** set up the service discovery internal exchanges ****
|
||||
# 1. declare RequestExchange to where to publish the RoutingData requests
|
||||
self.channel.exchange_declare(exchange=CM_REQUEST_EXCHANGE, exchange_type=ExchangeType.fanout)
|
||||
self.cm_request_exchange = await self.channel.declare_exchange(
|
||||
name=CM_REQUEST_EXCHANGE, type=ExchangeType.fanout
|
||||
)
|
||||
#
|
||||
# 2. declare ResponseExchange where to publish current RoutingData (response to
|
||||
# a RoutingData request)
|
||||
self.channel.exchange_declare(exchange=CM_RESPONSE_EXCHANGE, exchange_type=ExchangeType.fanout)
|
||||
self.cm_response_exchange = await self.channel.declare_exchange(
|
||||
name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout
|
||||
)
|
||||
# 2a. declare a Response queue for Routing Data replies from Services
|
||||
cm_response_queue: str = self.get_response_queue_name()
|
||||
self.channel.queue_declare(cm_response_queue, durable=True, exclusive=False, auto_delete=False,
|
||||
arguments={})
|
||||
response_queue: AbstractRobustQueue = await self.channel.declare_queue(
|
||||
name=cm_response_queue, durable=True, exclusive=False, auto_delete=False,
|
||||
arguments={}
|
||||
)
|
||||
# 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
|
||||
self.channel.queue_bind(queue=cm_response_queue, exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
|
||||
await response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
|
||||
# 2c. listen for incoming RoutingData messages
|
||||
self.channel.basic_consume(
|
||||
queue=cm_response_queue,
|
||||
auto_ack=False,
|
||||
on_message_callback=self.handle_routing_data_message)
|
||||
await response_queue.consume(callback=self.handle_routing_data_message, no_ack=False)
|
||||
logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
|
||||
except Exception as ioe:
|
||||
logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
|
||||
@@ -86,38 +81,34 @@ class RouterProducer(RouterBase):
|
||||
else:
|
||||
logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
|
||||
|
||||
def handle_routing_data_message(self,
|
||||
cb_channel: Channel,
|
||||
method: spec.Basic.Deliver,
|
||||
properties: spec.BasicProperties,
|
||||
body: bytes):
|
||||
async def handle_routing_data_message(self, message: AbstractIncomingMessage):
|
||||
"""
|
||||
* Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
|
||||
* routing data, invokes addRoute or removeRoute depending on the type of the message.
|
||||
"""
|
||||
logging.info(f" [x] Received {body}, m={method}, p={properties}")
|
||||
logging.info(f" [x] Received {message.body}, m={message.message_id}, p={message.properties}")
|
||||
"""
|
||||
logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
|
||||
delivery.getEnvelope().getDeliveryTag(),
|
||||
delivery.getEnvelope().toString(), str(delivery.getBody()))
|
||||
"""
|
||||
# ACK message now so that it is not being redelivered
|
||||
cb_channel.basic_ack(method.delivery_tag, False)
|
||||
message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
|
||||
await message.ack(multiple=False)
|
||||
cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
|
||||
# ignore the message if it comes from ourselves (even if from different instance)
|
||||
logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
|
||||
method.delivery_tag,
|
||||
self.service_message_factory.to_string(message),
|
||||
message.delivery_tag,
|
||||
self.service_message_factory.to_string(cm_message),
|
||||
self.amq_configuration.amq_adapter.route_mapping,
|
||||
self.amq_configuration.amq_adapter.service_name,
|
||||
message.reply_to)
|
||||
if message.is_valid() and not self.amq_configuration.amq_adapter.service_name == message.reply_to:
|
||||
if message.message_type == ServiceMessageType.ROUTING_DATA:
|
||||
self.add_routes(message.message)
|
||||
if message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
|
||||
self.remove_routes(message.message)
|
||||
cm_message.reply_to)
|
||||
if cm_message.is_valid() and not self.amq_configuration.amq_adapter.service_name == cm_message.reply_to:
|
||||
if cm_message.message_type == ServiceMessageType.ROUTING_DATA:
|
||||
self.add_routes(cm_message.message)
|
||||
if cm_message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
|
||||
self.remove_routes(cm_message.message)
|
||||
|
||||
def request_routes_from_adapters(self):
|
||||
async def request_routes_from_adapters(self):
|
||||
"""
|
||||
* Send a broadcast message prompting all listening adapters to reply with own routing data.
|
||||
"""
|
||||
@@ -126,37 +117,37 @@ class RouterProducer(RouterBase):
|
||||
serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
|
||||
ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
|
||||
)
|
||||
if not self.publish_service_message(serviceMessage, CM_REQUEST_EXCHANGE):
|
||||
if not await self.publish_service_message(serviceMessage, self.cm_request_exchange):
|
||||
logging.warning(" [E] RouterProducer could not request route data: %s channel is not open.",
|
||||
CM_REQUEST_EXCHANGE)
|
||||
|
||||
except Exception as ioe:
|
||||
logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
|
||||
|
||||
BASIC: BasicProperties = BasicProperties(content_type="application/octet-stream",
|
||||
content_encoding=None,
|
||||
headers=None,
|
||||
delivery_mode=1,
|
||||
priority=0,
|
||||
correlation_id=None)
|
||||
|
||||
def publish_service_message(self, message: CleverMicroServiceMessage, exchange_name: str) -> bool:
|
||||
async def publish_service_message(self,
|
||||
message: CleverMicroServiceMessage,
|
||||
exchange: AbstractRobustExchange) -> bool:
|
||||
"""
|
||||
* Publishes a service message to the service queue and logs success log entry.
|
||||
*
|
||||
* :param message: Service Message to be sent
|
||||
* :param exchangeName: target exchange name where to publish the message
|
||||
* :param queueName target: queue name (Optional, purpose is to get the queue size only)
|
||||
* :param exchange: target exchange name where to publish the message
|
||||
* :return True if channel is open, False otherwise
|
||||
* @throws IOException if something else goes wrong
|
||||
"""
|
||||
if self.is_open(self.channel):
|
||||
binary_content: bytes = self.service_message_factory.serialize(message)
|
||||
self.channel.basic_publish(exchange=exchange_name,
|
||||
routing_key="", # empty routing key
|
||||
properties=self.BASIC,
|
||||
body=binary_content)
|
||||
logging.info(" [x] Service Message Sent to %s, msg: %s", exchange_name, str(binary_content))
|
||||
pika_message: Message = Message(
|
||||
body=binary_content,
|
||||
content_encoding='utf-8',
|
||||
delivery_mode=2,
|
||||
content_type="application/octet-stream",
|
||||
headers=None,
|
||||
priority=0,
|
||||
correlation_id=None
|
||||
)
|
||||
await exchange.publish(message=pika_message, routing_key="")
|
||||
logging.info(" [x] Service Message Sent to %s, msg: %s", exchange.name, str(binary_content))
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -181,10 +172,10 @@ class RouterProducer(RouterBase):
|
||||
"""
|
||||
return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
|
||||
|
||||
def is_open(self, _channel: Channel) -> bool:
|
||||
def is_open(self, _channel: AbstractRobustChannel) -> bool:
|
||||
"""
|
||||
* Test if the channel is usable (opened).
|
||||
* :param _channel: the channel
|
||||
* :return True if the channel is opened.
|
||||
"""
|
||||
return _channel is not None and _channel.is_open
|
||||
return _channel is not None and not _channel.is_closed
|
||||
|
||||
Reference in New Issue
Block a user