clevermicro/amq-adapter-python
14
0
Fork 0
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Add IDLE backpressure detection

Browse Source
This commit is contained in:
Stanislav Hejny
2025-04-06 01:24:51 +01:00
parent f7bffbe20d
commit fc110c7175
36 changed files with 904 additions and 1050 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amqp/router/router_base.py
+3 -3
View File
@@ -2,7 +2,7 @@ import logging
from typing import Callable, List, Set
from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
from amqp.model.model import AMQRoute, AMQMessage
from amqp.model.model import AMQRoute, DataMessage
from amqp.router.route_database import RouteDatabase
from amqp.router.utils import sanitize_as_rabbitmq_name
@@ -26,7 +26,7 @@ class RouterBase:
self.route_added_notifier: Callable[[AMQRoute], None] | None = None
self.route_removed_notifier: Callable[[AMQRoute], None] | None = None
def get_routing_key(self, message: AMQMessage) -> str:
def get_routing_key(self, message: DataMessage) -> str:
return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
def wrap_routes(self) -> str:
@@ -46,7 +46,7 @@ class RouterBase:
def find_route(self, domain: str, path: str) -> AMQRoute | None:
return self.route_database.find_route(domain, path)
def find_route_by_message(self, message: AMQMessage) -> AMQRoute | None:
def find_route_by_message(self, message: DataMessage) -> AMQRoute | None:
return self.route_database.find_route(message.domain, message.path)
def add_routes(self, message: str) -> None:
amqp/router/router_consumer.py
+5 -5
View File
@@ -4,7 +4,7 @@ from typing import Set
from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractIncomingMessage
from amqp.adapter.amq_route_factory import AMQRouteFactory
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import CleverMicroServiceMessage, AMQRoute
from amqp.model.model import ServiceMessage, AMQRoute
from amqp.model.service_message_type import ServiceMessageType
from amqp.router.router_base import CM_C_REQ_QUEUE, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, ANY_RECIPIENT
from amqp.router.router_producer import RouterProducer
@@ -61,7 +61,7 @@ class RouterConsumer(RouterProducer):
await message.ack(False)
cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body)
route_mapping: str = self.wrap_own_routes()
logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
delivery_tag, self.service_message_factory.to_string(cm_message), route_mapping)
@@ -69,7 +69,7 @@ class RouterConsumer(RouterProducer):
if cm_message.is_valid() and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
if cm_message.reply_to != self.amq_configuration.amq_adapter.service_name:
try:
service_message: CleverMicroServiceMessage = self.service_message_factory.of(
service_message: ServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA, route_mapping, cm_message.reply_to
)
await self.publish_service_message(service_message, self.cm_response_exchange)
@@ -90,7 +90,7 @@ class RouterConsumer(RouterProducer):
"""
try:
if self.is_open(self.channel) and route.queue:
serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
serviceMessage: ServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA, route.as_string(), ANY_RECIPIENT
)
await self.publish_service_message(serviceMessage, self.cm_response_exchange)
@@ -110,7 +110,7 @@ class RouterConsumer(RouterProducer):
"""
logging.info(" [DBG] RouterConsumer.removeRoute, route: %s", route.as_string())
try:
service_message: CleverMicroServiceMessage = self.service_message_factory.of(
service_message: ServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA_REMOVE,
route.as_string(),
ANY_RECIPIENT
amqp/router/router_producer.py
+8 -8
View File
@@ -13,9 +13,9 @@ from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractRob
AbstractIncomingMessage
from pika.exchange_type import ExchangeType
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
from amqp.adapter.service_message_factory import ServiceMessageFactory
from amqp.config.amq_configuration import AMQConfiguration
from amqp.model.model import CleverMicroServiceMessage
from amqp.model.model import ServiceMessage
from amqp.model.service_message_type import ServiceMessageType
from amqp.router.router_base import RouterBase, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, CM_C_AMQ_QUEUE, \
CM_P_RESP_QUEUE, CM_P_REPLY_TO, ANY_RECIPIENT
@@ -30,7 +30,7 @@ class RouterProducer(RouterBase):
"""
super().__init__()
self.connection: AbstractRobustConnection | None = None
self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
self.service_message_factory = ServiceMessageFactory(configuration.amq_adapter.service_name)
self.amq_configuration: AMQConfiguration = configuration
self.channel: AbstractRobustChannel | None = None
self.cm_request_exchange: AbstractRobustExchange | None = None
@@ -40,9 +40,9 @@ class RouterProducer(RouterBase):
self.amq_configuration.amq_adapter.service_name,
self.amq_configuration.amq_adapter.route_mapping)
async def init_routing_paths(self, channel: AbstractRobustChannel | None):
async def init_internal_routing_paths(self, channel: AbstractRobustChannel | None):
"""
* Initialize router according to Producer mode.
* Initialize internal routes for ServiceMessages.
* The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
* created to enable the service discovery.
* The Producer mode sends out routing data request on CleverMicroRequest and listens
@@ -94,7 +94,7 @@ class RouterProducer(RouterBase):
"""
# ACK message now so that it is not being redelivered
await message.ack(multiple=False)
cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body)
# ignore the message if it comes from ourselves (even if from different instance)
logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
message.delivery_tag,
@@ -114,7 +114,7 @@ class RouterProducer(RouterBase):
"""
logging.info(" [*] RouterProducer.requestRoutesFromAdapters, queue name: %s", CM_REQUEST_EXCHANGE)
try:
serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
serviceMessage: ServiceMessage = self.service_message_factory.of(
ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
)
if not await self.publish_service_message(serviceMessage, self.cm_request_exchange):
@@ -125,7 +125,7 @@ class RouterProducer(RouterBase):
logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
async def publish_service_message(self,
message: CleverMicroServiceMessage,
message: ServiceMessage,
exchange: AbstractRobustExchange) -> bool:
"""
* Publishes a service message to the service queue and logs success log entry.
amqp/router/serializer.py
+4
View File
@@ -1,3 +1,7 @@
"""
Helper functions used to aid with (de)serialization of messages sent over AMQP.
"""
import struct
from io import BytesIO
from typing import List, Union, Dict
amqp/router/test_route_database.py
+2 -2
View File
@@ -2,7 +2,7 @@ from unittest import TestCase
import pytest
from amqp.adapter.amq_message_factory import AMQMessageFactory
from amqp.adapter.data_message_factory import DataMessageFactory
from amqp.adapter.amq_route_factory import AMQRouteFactory
from amqp.model.model import AMQRoute
from amqp.router.route_database import RouteDatabase
@@ -75,7 +75,7 @@ class TestRouteDatabase(TestCase):
self.assertTrue(ROUTE_2 in wrap)
def test_find_route(self):
message = AMQMessageFactory.get_instance(111).create_request_message(
message = DataMessageFactory.get_instance(111).create_request_message(
"POST", "clever3-book.localhost",
"/aa/bb?cc=d", {}, ""
)
amqp/router/test_router_base.py
+3 -3
View File
@@ -3,7 +3,7 @@ from unittest import TestCase
import pytest
from amqp.adapter.amq_message_factory import AMQMessageFactory
from amqp.adapter.data_message_factory import DataMessageFactory
from amqp.adapter.amq_route_factory import AMQRouteFactory
from amqp.model.model import AMQRoute
from amqp.router.router_base import RouterBase
@@ -23,7 +23,7 @@ class TestRouterBase(TestCase):
@pytest.fixture(autouse=True)
def setup_each_test(self):
self.base = RouterBase()
self.factory = AMQMessageFactory.get_instance(111)
self.factory = DataMessageFactory.get_instance(111)
if len(self.base.get_routes()) == 0:
r1 = AMQRouteFactory.from_string(ROUTE_1)
r2 = AMQRouteFactory.from_string(ROUTE_2)
@@ -79,7 +79,7 @@ class TestRouterBase(TestCase):
self.assertIsNone(_route)
def test_find_route_by_message(self):
message = AMQMessageFactory.get_instance(111).create_request_message(
message = DataMessageFactory.get_instance(111).create_request_message(
"POST", "clever3-book.localhost",
"/aa/bb?cc=d", {}, ""
)