clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

AMQ adapter version w/o endpints for CleverSwarm integration

Browse Source
This commit is contained in:
Stanislav Hejny
2025-01-23 10:27:21 +00:00
committed by Rui Hu
parent d2b1b59c66
commit 286a4986b3
39 changed files with 1104 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/router/__init__.py
View File
amqp/router/router_base.py
+2 -2
View File
@@ -74,9 +74,9 @@ class RouterBase:
try:
to_remove = [AMQRouteFactory.from_string(route_str) for route_str in message.split(",")]
removed_count = sum(1 for route in to_remove if self.remove_route(route))
print(f" [*] RouterMaster.removeRoutes(): {message}, removed={removed_count}")
logging.info(f" [*] RouterMaster.removeRoutes(): {message}, removed={removed_count}")
except IOError as err:
print(f" [E] Can't remove route(s): {err}")
logging.info(f" [E] Can't remove route(s): {err}")
def remove_route(self, route: AMQRoute) -> bool:
is_route_removed = self.route_database.remove_route(route)
amqp/router/router_consumer.py
+12 -11
View File
@@ -2,11 +2,12 @@ import logging
from typing import Set
from pika import spec
from pika.adapters.blocking_connection import BlockingChannel
from pika.channel import Channel
from amqp.adapter.amq_route_factory import AMQRouteFactory
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType, AMQRoute
from amqp.model.model import CleverMicroServiceMessage, AMQRoute
from amqp.model.service_message_type import ServiceMessageType
from amqp.router.router_base import CM_C_REQ_QUEUE, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, ANY_RECIPIENT
from amqp.router.router_producer import RouterProducer
@@ -29,13 +30,13 @@ class RouterConsumer(RouterProducer):
self.amqRouteFactory = AMQRouteFactory()
logging.info(" [*] RouterConsumer.<init>")
def init_routing_paths_consumer(self, _channel: BlockingChannel):
def init_routing_paths_consumer(self, _channel: Channel):
"""
* Initialize router according to Consumer mode.
"""
self.init_routing_paths(_channel)
logging.info(" [*] RouterConsumer.initConsumerRouter, channel open::", self.is_open(_channel))
self.init_routing_paths_consumer(_channel)
logging.info(" [*] RouterConsumer.initConsumerRouter, channel open: %s", self.is_open(_channel))
if self.is_open(_channel):
try:
# 1. declare a Queue that gives the RoutingDAta requests
@@ -44,7 +45,7 @@ class RouterConsumer(RouterProducer):
# 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
self.channel.queue_bind(queue=cm_request_queue, exchange=CM_REQUEST_EXCHANGE, routing_key="")
# 1b. Service adapter mode - listens on Request queue and replies to Response queue
logging.info(" [*] RouterConsumer listens for RouteDataRequests on:", cm_request_queue)
logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue)
self.channel.basic_consume(queue=cm_request_queue,
auto_ack=False,
on_message_callback=self.handle_routing_data_request_message)
@@ -58,7 +59,7 @@ class RouterConsumer(RouterProducer):
"""
def handle_routing_data_request_message(self,
ch: BlockingChannel,
cb_channel: Channel,
method: spec.Basic.Deliver,
properties: spec.BasicProperties,
body: bytes):
@@ -68,11 +69,11 @@ class RouterConsumer(RouterProducer):
logging.info(" [DBG] ******[%] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
delivery_tag, method.consumer_tag, method.get_properties(), debug)
self.channel.basic_ack(delivery_tag, False)
cb_channel.basic_ack(delivery_tag, False)
message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
route_mapping: str = self.wrap_own_routes()
logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%, routeMap=[{]",
logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
delivery_tag, self.service_message_factory.to_string(message), route_mapping)
# Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
if message.is_valid() and message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
@@ -102,7 +103,7 @@ class RouterConsumer(RouterProducer):
)
self.publish_service_message(serviceMessage, CM_RESPONSE_EXCHANGE)
self.add_own_route(route)
logging.info(" [DBG] RouterConsumer registered Route::", route)
logging.info(" [DBG] RouterConsumer registered Route: %s", route)
else:
logging.warning(" [W] RouterConsumer couldn't register route data: %s channel isn't open.",
CM_RESPONSE_EXCHANGE)
@@ -115,7 +116,7 @@ class RouterConsumer(RouterProducer):
* :param route: route to remove
* :return result of remove operation
"""
logging.info(" [DBG] RouterConsumer.removeRoute, route::", route.as_string())
logging.info(" [DBG] RouterConsumer.removeRoute, route: %s", route.as_string())
try:
service_message: CleverMicroServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA_REMOVE,
amqp/router/router_producer.py
+13 -13
View File
@@ -10,12 +10,13 @@ import logging
import os
import pika
from pika import BasicProperties, spec
from pika.adapters.blocking_connection import BlockingChannel
from pika.channel import Channel
from pika.exchange_type import ExchangeType
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import CleverMicroServiceMessage, ServiceMessageType
from amqp.model.model import CleverMicroServiceMessage
from amqp.model.service_message_type import ServiceMessageType
from amqp.router.router_base import RouterBase, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, CM_C_AMQ_QUEUE, \
CM_P_RESP_QUEUE, CM_P_REPLY_TO, ANY_RECIPIENT
@@ -31,7 +32,7 @@ class RouterProducer(RouterBase):
self.connection = None
self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
self.amq_configuration: AMQConfiguration = configuration
self.channel: pika.adapters.blocking_connection.BlockingChannel | None = None
self.channel: Channel | None = None
logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
self.amq_configuration.amq_adapter.service_name,
@@ -41,13 +42,13 @@ class RouterProducer(RouterBase):
credentials = pika.PlainCredentials(
os.environ['RABBIT_MQ_USER'], os.environ['RABBIT_MQ_PASSWORD']
)
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost', credentials=credentials)
self.connection: pika.SelectConnection = pika.SelectConnection(
pika.ConnectionParameters(host=self.amq_configuration.dispatch.amq_host, credentials=credentials)
)
self.channel = self.connection.channel()
self.channel.start_consuming()
# self.channel.basic_consume()
def init_routing_paths(self, channel: pika.adapters.blocking_connection.BlockingChannel | None):
def init_routing_paths(self, channel: Channel | None):
"""
* Initialize router according to Producer mode.
* The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
@@ -78,7 +79,6 @@ class RouterProducer(RouterBase):
queue=cm_response_queue,
auto_ack=False,
on_message_callback=self.handle_routing_data_message)
self.channel.start_consuming()
logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
except Exception as ioe:
logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
@@ -87,7 +87,7 @@ class RouterProducer(RouterBase):
logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
def handle_routing_data_message(self,
ch: BlockingChannel,
cb_channel: Channel,
method: spec.Basic.Deliver,
properties: spec.BasicProperties,
body: bytes):
@@ -95,17 +95,17 @@ class RouterProducer(RouterBase):
* Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
* routing data, invokes addRoute or removeRoute depending on the type of the message.
"""
print(f" [x] Received {body}, m={method}, p={properties}, ch={ch}")
logging.info(f" [x] Received {body}, m={method}, p={properties}")
"""
logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
delivery.getEnvelope().getDeliveryTag(),
delivery.getEnvelope().toString(), str(delivery.getBody()))
"""
# ACK message now so that it is not being redelivered
self.channel.basic_ack(method.delivery_tag, False)
cb_channel.basic_ack(method.delivery_tag, False)
message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(body)
# ignore the message if it comes from ourselves (even if from different instance)
logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[{] sn={, replyTo={",
logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
method.delivery_tag,
self.service_message_factory.to_string(message),
self.amq_configuration.amq_adapter.route_mapping,
@@ -181,7 +181,7 @@ class RouterProducer(RouterBase):
"""
return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
def is_open(self, _channel: pika.adapters.blocking_connection.BlockingChannel) -> bool:
def is_open(self, _channel: Channel) -> bool:
"""
* Test if the channel is usable (opened).
* :param _channel: the channel
amqp/router/test_utils.py
+2 -1
View File
@@ -10,6 +10,7 @@ class Test(TestCase):
self.assertEqual("aa", sanitize_as_rabbitmq_name("aa&bb/cc"))
self.assertEqual("aa", sanitize_as_rabbitmq_name("aa+bb/cc"))
self.assertEqual("aa", sanitize_as_rabbitmq_name("aa%bb/cc"))
self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa#bb/cc"))
self.assertEqual("aa#bb.cc", sanitize_as_rabbitmq_name("aa#bb/cc"))
self.assertEqual("aa#bb.cc.#", sanitize_as_rabbitmq_name("aa#bb/cc.#"))
self.assertEqual("aa", sanitize_as_rabbitmq_name("aa?bb////cc"))
self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa///bb////cc"))
amqp/router/utils.py
+1 -1
View File
@@ -5,5 +5,5 @@ def sanitize_as_rabbitmq_name(input_str: str) -> str:
pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
matcher = pattern.search(input_str)
token = input_str[:matcher.start()] if matcher else input_str
step1 = re.sub(r"[^A-Za-z0-9]", ".", token)
step1 = re.sub(r"[^A-Za-z0-9#]", ".", token)
return re.sub(r"\.+", ".", step1)