restructure to fit around python event-loop
This commit is contained in:
@@ -9,12 +9,12 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
import pathlib
|
import pathlib
|
||||||
from asyncio import Future
|
from asyncio import Future, AbstractEventLoop
|
||||||
from typing import Dict, List, Tuple, Callable, Coroutine, Any
|
from typing import Dict, List, Tuple, Callable, Coroutine, Any
|
||||||
from urllib.parse import parse_qs, urlparse
|
from urllib.parse import parse_qs, urlparse
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
from aio_pika.abc import AbstractRobustChannel
|
from aio_pika.abc import AbstractRobustChannel, AbstractRobustConnection, AbstractRobustExchange, AbstractExchange
|
||||||
from aiohttp import ClientSession, ClientResponse
|
from aiohttp import ClientSession, ClientResponse
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
@@ -28,6 +28,8 @@ from amqp.adapter.file_uploader import publish_file_to_rabbitmq
|
|||||||
|
|
||||||
from IPython.core import debugger
|
from IPython.core import debugger
|
||||||
|
|
||||||
|
from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
|
||||||
|
|
||||||
debug = debugger.Pdb().set_trace
|
debug = debugger.Pdb().set_trace
|
||||||
|
|
||||||
|
|
||||||
@@ -38,7 +40,7 @@ class AMQMessage(DataMessage):
|
|||||||
in case the response should initiate file download.
|
in case the response should initiate file download.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, data_message: DataMessage, channel: AbstractRobustChannel):
|
def __init__(self, data_message: DataMessage, connection: AbstractRobustConnection):
|
||||||
# Copy all fields from DataMessage
|
# Copy all fields from DataMessage
|
||||||
self.magic = data_message.magic
|
self.magic = data_message.magic
|
||||||
self.id = data_message.id
|
self.id = data_message.id
|
||||||
@@ -49,7 +51,7 @@ class AMQMessage(DataMessage):
|
|||||||
self.trace_info = data_message.trace_info
|
self.trace_info = data_message.trace_info
|
||||||
self.base64_body = data_message.base64_body
|
self.base64_body = data_message.base64_body
|
||||||
# Add the new field
|
# Add the new field
|
||||||
self.channel = channel
|
self.connection = connection
|
||||||
|
|
||||||
|
|
||||||
def parse_request_body(message: DataMessage):
|
def parse_request_body(message: DataMessage):
|
||||||
@@ -115,11 +117,35 @@ def parse_request_body(message: DataMessage):
|
|||||||
raise ValueError(f"Unsupported Content-Type: {content_type}")
|
raise ValueError(f"Unsupported Content-Type: {content_type}")
|
||||||
|
|
||||||
|
|
||||||
async def convert_file_response(obj: FileResponse, channel: AbstractRobustChannel) -> str:
|
async def convert_file_response(
|
||||||
|
obj: FileResponse,
|
||||||
|
connection: AbstractRobustConnection,
|
||||||
|
loop: AbstractEventLoop | None
|
||||||
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Converts a FileResponse object to a JSON string containing the filename and stream.
|
Converts a FileResponse object to a JSON string containing the filename and stream.
|
||||||
"""
|
"""
|
||||||
(_stream_name, _file_size) = await publish_file_to_rabbitmq(channel, obj.path)
|
async def wrap_connection_channel(connection: AbstractRobustConnection) -> tuple[AbstractRobustChannel, AbstractExchange, str]:
|
||||||
|
""" Wrap call to channel() to make it Awaitable """
|
||||||
|
_channel = await connection.channel()
|
||||||
|
exchange_name: str = AMQ_REPLY_TO_EXCHANGE
|
||||||
|
_exchange = await _channel.get_exchange(name=exchange_name, ensure=True)
|
||||||
|
_queue = await _channel.declare_queue(exclusive=True) # Declare a unique, exclusive queue
|
||||||
|
_queue_name = _queue.name
|
||||||
|
await _queue.bind(
|
||||||
|
exchange=exchange_name,
|
||||||
|
routing_key=_queue_name, # Use queue name as routing key
|
||||||
|
)
|
||||||
|
|
||||||
|
logging.debug(f" [*] Publishing to exchange: {exchange_name}, q: {_queue_name}")
|
||||||
|
return _channel, _exchange, _queue_name
|
||||||
|
|
||||||
|
_future = asyncio.run_coroutine_threadsafe(wrap_connection_channel(connection), loop)
|
||||||
|
while not _future.done():
|
||||||
|
await asyncio.sleep(0.1) # allow coroutine to execute
|
||||||
|
_channel, _exchange, _routing_key = _future.result()
|
||||||
|
(_stream_name, _file_size) = await publish_file_to_rabbitmq(_exchange, obj.path, _routing_key, loop)
|
||||||
|
await _channel.close()
|
||||||
return json.dumps({'files': [
|
return json.dumps({'files': [
|
||||||
{'filename': obj.filename,
|
{'filename': obj.filename,
|
||||||
'mediaType': 'application/octet-stream',
|
'mediaType': 'application/octet-stream',
|
||||||
@@ -133,7 +159,8 @@ async def call_cleverthis_api(
|
|||||||
message: AMQMessage,
|
message: AMQMessage,
|
||||||
args: Any,
|
args: Any,
|
||||||
api_method: Callable[[Any, Any], Coroutine],
|
api_method: Callable[[Any, Any], Coroutine],
|
||||||
success_code: int
|
success_code: int,
|
||||||
|
loop: AbstractEventLoop | None = None
|
||||||
) -> DataResponse:
|
) -> DataResponse:
|
||||||
"""
|
"""
|
||||||
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
|
Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
|
||||||
@@ -162,7 +189,7 @@ async def call_cleverthis_api(
|
|||||||
|
|
||||||
if isinstance(_result, FileResponse):
|
if isinstance(_result, FileResponse):
|
||||||
_json_result, _content_type = await convert_file_response(
|
_json_result, _content_type = await convert_file_response(
|
||||||
_result, message.channel
|
_result, message.connection, loop
|
||||||
), 'application/octet-stream'
|
), 'application/octet-stream'
|
||||||
else:
|
else:
|
||||||
_json_result = _result
|
_json_result = _result
|
||||||
@@ -224,6 +251,7 @@ class CleverThisServiceAdapter:
|
|||||||
self.session = session
|
self.session = session
|
||||||
self.service_host = self.amq_configuration.dispatch.service_host
|
self.service_host = self.amq_configuration.dispatch.service_host
|
||||||
self.service_port = self.amq_configuration.dispatch.service_port
|
self.service_port = self.amq_configuration.dispatch.service_port
|
||||||
|
self.loop: AbstractEventLoop | None = None
|
||||||
self.require_authenticated_user = eval(
|
self.require_authenticated_user = eval(
|
||||||
self.amq_configuration.amq_adapter.require_authenticated_user.capitalize()) if self.amq_configuration.amq_adapter.require_authenticated_user.strip() else True
|
self.amq_configuration.amq_adapter.require_authenticated_user.capitalize()) if self.amq_configuration.amq_adapter.require_authenticated_user.strip() else True
|
||||||
|
|
||||||
@@ -247,7 +275,7 @@ class CleverThisServiceAdapter:
|
|||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def on_message(self, message: AMQMessage, future: Future[DataResponse] = None) -> DataResponse:
|
async def on_message(self, message: AMQMessage) -> DataResponse:
|
||||||
"""
|
"""
|
||||||
Called when a message is received from the AMQP.
|
Called when a message is received from the AMQP.
|
||||||
:param message:
|
:param message:
|
||||||
@@ -292,9 +320,6 @@ class CleverThisServiceAdapter:
|
|||||||
raise ValueError(f"Unexpected HTTP method: {message.method}")
|
raise ValueError(f"Unexpected HTTP method: {message.method}")
|
||||||
else:
|
else:
|
||||||
amq_response = DataResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
|
amq_response = DataResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
|
||||||
|
|
||||||
if future:
|
|
||||||
future.set_result(amq_response)
|
|
||||||
return amq_response
|
return amq_response
|
||||||
|
|
||||||
# ==================================================================================================
|
# ==================================================================================================
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
import logging
|
import logging
|
||||||
|
from asyncio import AbstractEventLoop
|
||||||
|
|
||||||
import aio_pika
|
import aio_pika
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from aio_pika.abc import AbstractRobustChannel
|
from aio_pika.abc import AbstractRobustChannel, AbstractRobustExchange
|
||||||
from aio_pika import Message, DeliveryMode
|
from aio_pika import Message, DeliveryMode
|
||||||
|
|
||||||
from amqp.adapter.file_downloader import IN_PROGRESS, END_OF_FILE
|
from amqp.adapter.file_downloader import IN_PROGRESS, END_OF_FILE
|
||||||
@@ -12,10 +14,11 @@ from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
|
|||||||
|
|
||||||
|
|
||||||
async def publish_file_to_rabbitmq(
|
async def publish_file_to_rabbitmq(
|
||||||
channel: AbstractRobustChannel,
|
exchange: AbstractRobustExchange,
|
||||||
file_path: Path,
|
file_path: Path,
|
||||||
|
routing_key: str,
|
||||||
|
loop: AbstractEventLoop,
|
||||||
max_chunk_size: int = 2**20, # 1MB default chunk size
|
max_chunk_size: int = 2**20, # 1MB default chunk size
|
||||||
exchange_name: str = AMQ_REPLY_TO_EXCHANGE,
|
|
||||||
content_type: str = 'application/octet-stream', # Default content type
|
content_type: str = 'application/octet-stream', # Default content type
|
||||||
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
|
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
|
||||||
) -> tuple[str, int]:
|
) -> tuple[str, int]:
|
||||||
@@ -25,10 +28,12 @@ async def publish_file_to_rabbitmq(
|
|||||||
exchange, and uses the queue name as the routing key.
|
exchange, and uses the queue name as the routing key.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
channel: The aio_pika RobustChannel to use for communication with RabbitMQ.
|
exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
|
||||||
file_path: The path to the file to publish.
|
file_path: The path to the file to publish.
|
||||||
max_chunk_size: The maximum size of each chunk (in bytes).
|
max_chunk_size: The maximum size of each chunk (in bytes).
|
||||||
exchange_name: The name of the RabbitMQ exchange to publish to.
|
routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
|
||||||
|
loop: correct event loop
|
||||||
|
max_chunk_size: chunk size to send
|
||||||
content_type: The content type of the file data.
|
content_type: The content type of the file data.
|
||||||
delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
|
delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
|
||||||
|
|
||||||
@@ -38,15 +43,6 @@ async def publish_file_to_rabbitmq(
|
|||||||
if not file_path.is_file():
|
if not file_path.is_file():
|
||||||
raise FileNotFoundError(f"File not found: {file_path}")
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
_file_size = os.path.getsize(file_path)
|
_file_size = os.path.getsize(file_path)
|
||||||
_exchange = await channel.get_exchange(name=exchange_name, ensure=True)
|
|
||||||
_queue = await channel.declare_queue(exclusive=True) # Declare a unique, exclusive queue
|
|
||||||
_queue_name = _queue.name
|
|
||||||
await _queue.bind(
|
|
||||||
exchange=exchange_name,
|
|
||||||
routing_key=_queue_name, # Use queue name as routing key
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.info(f"Publishing file: {file_path} (size: {_file_size}) to exchg: {exchange_name}, q: {_queue_name}")
|
|
||||||
|
|
||||||
with open(file_path, 'rb') as f:
|
with open(file_path, 'rb') as f:
|
||||||
_offset = 0
|
_offset = 0
|
||||||
@@ -67,9 +63,12 @@ async def publish_file_to_rabbitmq(
|
|||||||
'eof': END_OF_FILE if max_chunk_size + _offset >= _file_size else IN_PROGRESS
|
'eof': END_OF_FILE if max_chunk_size + _offset >= _file_size else IN_PROGRESS
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
await _exchange.publish(message, routing_key=_queue_name)
|
_future = asyncio.run_coroutine_threadsafe(exchange.publish(message, routing_key=routing_key), loop)
|
||||||
|
while not _future.done():
|
||||||
|
# sleep allows coroutine to execute. must wait for completion to ensure all chunks written in right order
|
||||||
|
await asyncio.sleep(0.02)
|
||||||
_offset += len(_chunk)
|
_offset += len(_chunk)
|
||||||
logging.debug(f"Published chunk {_offset}/{_file_size} to {exchange_name}:{_queue_name}")
|
logging.debug(f"Published chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}")
|
||||||
|
|
||||||
print(f"File {file_path} published to RabbitMQ exchange {exchange_name}, queue {_queue_name}")
|
print(f"File {file_path} published to RabbitMQ exchange {exchange.name}, queue {routing_key}")
|
||||||
return _queue_name, _file_size
|
return routing_key, _file_size
|
||||||
|
|||||||
@@ -34,13 +34,11 @@ last_data_message_time = 0 # helps detect IDLE condition
|
|||||||
last_backpressure_event_time = 0 # helps prevent flooding the system with backpressure events
|
last_backpressure_event_time = 0 # helps prevent flooding the system with backpressure events
|
||||||
last_backpressure_event = ScalingRequestAlert.IDLE
|
last_backpressure_event = ScalingRequestAlert.IDLE
|
||||||
|
|
||||||
# Global lock to synchronize access to the current_parallel_executions variable
|
|
||||||
lock = threading.Lock()
|
|
||||||
parallel_workers = int(os.getenv("PARALLEL_WORKERS", default=0))
|
parallel_workers = int(os.getenv("PARALLEL_WORKERS", default=0))
|
||||||
swarm_service_id = os.getenv("SWARM_SERVICE_ID", default="clever-amqp-service")
|
swarm_service_id = os.getenv("SWARM_SERVICE_ID", default="clever-amqp-service")
|
||||||
swarm_task_id = os.getenv("SWARM_TASK_ID", default="python-adapter")
|
swarm_task_id = os.getenv("SWARM_TASK_ID", default="python-adapter")
|
||||||
# Shared thread pool for parallel execution
|
# Shared thread pool for parallel execution
|
||||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers)
|
executor = concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers + 2)
|
||||||
|
|
||||||
|
|
||||||
def collect_trace_info():
|
def collect_trace_info():
|
||||||
@@ -79,6 +77,7 @@ class DataMessageHandler:
|
|||||||
self.rabbit_mq_url = amq_configuration.dispatch.rabbit_mq_url
|
self.rabbit_mq_url = amq_configuration.dispatch.rabbit_mq_url
|
||||||
self.output_dir = amq_configuration.dispatch.download_dir
|
self.output_dir = amq_configuration.dispatch.download_dir
|
||||||
self.loop = loop
|
self.loop = loop
|
||||||
|
self.service_adapter.loop = loop
|
||||||
# Dictionary to store outstanding Futures
|
# Dictionary to store outstanding Futures
|
||||||
self.outstanding: Dict[str, asyncio.Future] = {}
|
self.outstanding: Dict[str, asyncio.Future] = {}
|
||||||
self.reply_to: str | None = None
|
self.reply_to: str | None = None
|
||||||
@@ -133,10 +132,61 @@ class DataMessageHandler:
|
|||||||
)
|
)
|
||||||
await asyncio.sleep(self.backpressure_monitor_window) # sleep in seconds
|
await asyncio.sleep(self.backpressure_monitor_window) # sleep in seconds
|
||||||
|
|
||||||
def sync_on_message(self,
|
async def inbound_data_message_callback(self, message: AbstractIncomingMessage):
|
||||||
|
threading.Thread(target=lambda: asyncio.run(self.run_callback_thread(message))).start()
|
||||||
|
|
||||||
|
async def run_callback_thread(self, message: AbstractIncomingMessage):
|
||||||
|
"""
|
||||||
|
Receives the DataMessage as bytes from RabbitMQ queue and deserializes it to the DataMessage object
|
||||||
|
Ensures Jaeger trace-id propagation.
|
||||||
|
Invokes custom handler/mapper/adapter method that corresponds to this message
|
||||||
|
:param message:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
start = time.time()
|
||||||
|
amq_message = await self.reconstitute_data_message(message)
|
||||||
|
if amq_message is not None:
|
||||||
|
self.ensure_trace_info(amq_message)
|
||||||
|
if amq_message.magic == DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
|
||||||
|
await self.run_http_request_magic(message, amq_message, self.loop)
|
||||||
|
else:
|
||||||
|
logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
|
||||||
|
amq_message.magic)
|
||||||
|
asyncio.run_coroutine_threadsafe(message.nack(requeue=False), self.loop)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
logging.error(" [E] ######[%s] No valid AMQ Message present.", message.delivery_tag)
|
||||||
|
self.record_duration(start, message.delivery_tag)
|
||||||
|
asyncio.run_coroutine_threadsafe(message.ack(multiple=False), self.loop)
|
||||||
|
|
||||||
|
def handle_backpressure(self, _loop: AbstractEventLoop):
|
||||||
|
logging.warning("Warning: Capacity close to depleted!")
|
||||||
|
# Send a scaling request to the Management service
|
||||||
|
scaling_request: ScalingRequest = ScalingRequest(
|
||||||
|
swarm_service_id, swarm_task_id,
|
||||||
|
parallel_workers, parallel_workers - current_parallel_executions,
|
||||||
|
ScalingRequestAlert.OVERLOAD
|
||||||
|
)
|
||||||
|
# Address the message to any adapter capable of supporting BACKPRESSURE request
|
||||||
|
scale_up_request = self.service_message_factory.of(
|
||||||
|
ServiceMessageType.BACKPRESSURE,
|
||||||
|
scaling_request.to_json(),
|
||||||
|
self.reply_to if self.reply_to else ANY_RECIPIENT
|
||||||
|
)
|
||||||
|
asyncio.run_coroutine_threadsafe(
|
||||||
|
self.rabbit_mq_client.router.publish_service_message(
|
||||||
|
scale_up_request, exchange=self.rabbit_mq_client.router.cm_response_exchange
|
||||||
|
),
|
||||||
|
loop=_loop
|
||||||
|
)
|
||||||
|
# update / reset time-window so that the OVERLOAD is not sent too often
|
||||||
|
global last_data_message_time
|
||||||
|
last_datadeliveryTag_message_time = time.time()
|
||||||
|
|
||||||
|
async def run_http_request_magic(self,
|
||||||
message: AbstractIncomingMessage,
|
message: AbstractIncomingMessage,
|
||||||
amq_message: DataMessage,
|
amq_message: DataMessage,
|
||||||
loop: AbstractEventLoop) -> None:
|
_loop: AbstractEventLoop) -> None:
|
||||||
"""
|
"""
|
||||||
Synchronous version of on_message method. This is a wrapper around the async on_message method.
|
Synchronous version of on_message method. This is a wrapper around the async on_message method.
|
||||||
It also counts parallel executions and handles backpressure.
|
It also counts parallel executions and handles backpressure.
|
||||||
@@ -148,45 +198,17 @@ class DataMessageHandler:
|
|||||||
global current_parallel_executions
|
global current_parallel_executions
|
||||||
|
|
||||||
# Increment the counter for parallel executions
|
# Increment the counter for parallel executions
|
||||||
with lock:
|
current_parallel_executions += 1
|
||||||
current_parallel_executions += 1
|
logging.info(f"PARALLEL: Current Capacity [{current_parallel_executions}] of [{parallel_workers}]")
|
||||||
logging.info(f"PARALLEL: Current Capacity [{current_parallel_executions}] of [{parallel_workers}]")
|
if message.reply_to:
|
||||||
if message.reply_to:
|
self.reply_to = message.reply_to
|
||||||
self.reply_to = message.reply_to
|
if current_parallel_executions > parallel_workers - 2:
|
||||||
if current_parallel_executions > parallel_workers - 2:
|
self.handle_backpressure(_loop)
|
||||||
logging.warning("Warning: Capacity close to depleted!")
|
|
||||||
# Send a scaling request to the Management service
|
|
||||||
scaling_request: ScalingRequest = ScalingRequest(
|
|
||||||
swarm_service_id, swarm_task_id,
|
|
||||||
parallel_workers, parallel_workers - current_parallel_executions,
|
|
||||||
ScalingRequestAlert.OVERLOAD
|
|
||||||
)
|
|
||||||
# Address the message to any adapter capable of supporting BACKPRESSURE request
|
|
||||||
scale_up_request = self.service_message_factory.of(
|
|
||||||
ServiceMessageType.BACKPRESSURE,
|
|
||||||
scaling_request.to_json(),
|
|
||||||
self.reply_to if self.reply_to else ANY_RECIPIENT
|
|
||||||
)
|
|
||||||
asyncio.run_coroutine_threadsafe(
|
|
||||||
self.rabbit_mq_client.router.publish_service_message(
|
|
||||||
scale_up_request, exchange=self.rabbit_mq_client.router.cm_response_exchange
|
|
||||||
),
|
|
||||||
loop=loop
|
|
||||||
)
|
|
||||||
# update / reset time-window so that the OVERLOAD is not sent too often
|
|
||||||
global last_data_message_time
|
|
||||||
last_data_message_time = time.time()
|
|
||||||
|
|
||||||
# Call the async on_message method. This shall also ACK the message after processing.
|
try:
|
||||||
a_message = AMQMessage(amq_message, self.rabbit_mq_client.channel)
|
a_message = AMQMessage(amq_message, self.rabbit_mq_client.connection)
|
||||||
future: Future[DataResponse] = asyncio.Future(loop=loop)
|
response: DataResponse = await self.service_adapter.on_message(a_message)
|
||||||
print(f" [*] ######[{message.delivery_tag}] about to call asyncio.run(on_message) with AMQ Message: {a_message}")
|
print(f"Result in thread: {response}")
|
||||||
ftr: Future[DataResponse] = asyncio.run_coroutine_threadsafe(self.service_adapter.on_message(a_message, future), loop)
|
|
||||||
print(f"WILL AWAIT RESP {ftr} -> {ftr.done()}")
|
|
||||||
|
|
||||||
async def wait_for_response(future: Future):
|
|
||||||
print(f"AWAITINMG RESPONSE %s -> %s", future, future.done())
|
|
||||||
response: DataResponse = future.result()
|
|
||||||
try:
|
try:
|
||||||
logging.info(" [*] ######[%s] AMQ Message Reply.1, To=%s, msgId=%s",
|
logging.info(" [*] ######[%s] AMQ Message Reply.1, To=%s, msgId=%s",
|
||||||
message.delivery_tag, message.reply_to, response.id)
|
message.delivery_tag, message.reply_to, response.id)
|
||||||
@@ -197,7 +219,7 @@ class DataMessageHandler:
|
|||||||
logging.info(f"[E] Error processing message: {e}")
|
logging.info(f"[E] Error processing message: {e}")
|
||||||
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
|
tb = traceback.extract_tb(sys.exc_info()[2])[-1]
|
||||||
logging.error(f"[E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
|
logging.error(f"[E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
|
||||||
asyncio.run_coroutine_threadsafe(message.nack(requeue=False), loop=loop)
|
asyncio.run_coroutine_threadsafe(message.nack(requeue=False), loop=_loop)
|
||||||
|
|
||||||
if isinstance(amq_message, MultipartDataMessage):
|
if isinstance(amq_message, MultipartDataMessage):
|
||||||
# remove the downloaded files from the output_dir
|
# remove the downloaded files from the output_dir
|
||||||
@@ -207,94 +229,59 @@ class DataMessageHandler:
|
|||||||
os.remove(filename)
|
os.remove(filename)
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
logging.error(f"Error deleting file {filename}: {e}")
|
logging.error(f"Error deleting file {filename}: {e}")
|
||||||
|
current_parallel_executions -= 1
|
||||||
with lock:
|
return
|
||||||
current_parallel_executions -= 1
|
except Exception as e:
|
||||||
print(f"KICK OFF wait proc {future}")
|
print(f"Error in thread: {e}")
|
||||||
asyncio.run_coroutine_threadsafe(wait_for_response(future), loop)
|
return
|
||||||
|
|
||||||
|
|
||||||
async def inbound_data_message_callback(self, message: AbstractIncomingMessage):
|
async def reconstitute_data_message(self, message: AbstractIncomingMessage) -> DataMessage | None:
|
||||||
"""
|
|
||||||
Receives the DataMessage as bytes from RabbitMQ queue and deserializes it to the DataMessage object
|
|
||||||
Ensures Jaeger trace-id propagation.
|
|
||||||
Invokes custom handler/mapper/adapter method that corresponds to this message
|
|
||||||
:param message:
|
|
||||||
:return:
|
|
||||||
"""
|
|
||||||
start = time.time()
|
|
||||||
delivery_tag = message.delivery_tag
|
|
||||||
debug = f" [DBG] ######[{delivery_tag}] (AMQ MESSAGE), env: {message.headers}, props: {message.properties}"
|
|
||||||
logging.debug(debug)
|
|
||||||
addressing: int = message.headers.get('clevermicro.addressing', 0)
|
|
||||||
|
|
||||||
amq_message: DataMessage | None = None
|
amq_message: DataMessage | None = None
|
||||||
|
delivery_tag = message.delivery_tag
|
||||||
|
addressing: int = message.headers.get('clevermicro.addressing', 0)
|
||||||
if addressing == 0:
|
if addressing == 0:
|
||||||
amq_message = DataMessageFactory.from_bytes(message.body)
|
amq_message = DataMessageFactory.from_bytes(message.body)
|
||||||
else:
|
else:
|
||||||
loop = asyncio.get_event_loop()
|
amq_message = await DataMessageFactory.from_stream(message.body, rabbitmq_url=self.rabbit_mq_url, loop=self.loop)
|
||||||
amq_message = await DataMessageFactory.from_stream(message.body, rabbitmq_url=self.rabbit_mq_url, loop=loop)
|
|
||||||
logging.info(f" [DBG] ######[{delivery_tag}] AMQ Message from stream: {amq_message}")
|
logging.info(f" [DBG] ######[{delivery_tag}] AMQ Message from stream: {amq_message}")
|
||||||
if amq_message is not None:
|
if amq_message is not None:
|
||||||
extra_data = await on_message(
|
extra_data = await on_message(
|
||||||
message=message,
|
message=message,
|
||||||
json_data=amq_message.body(),
|
json_data=amq_message.body(),
|
||||||
rabbitmq_url=self.rabbit_mq_url,
|
rabbitmq_url=self.rabbit_mq_url,
|
||||||
loop=loop,
|
loop=self.loop,
|
||||||
output_dir=self.output_dir)
|
output_dir=self.output_dir)
|
||||||
logging.info(f" [x] ######[{delivery_tag}] Multipart download result: {extra_data}")
|
logging.info(f" [x] ######[{delivery_tag}] Multipart download result: {extra_data}")
|
||||||
amq_message = MultipartDataMessage(amq_message, extra_data)
|
amq_message = MultipartDataMessage(amq_message, extra_data)
|
||||||
|
if amq_message:
|
||||||
if amq_message is not None:
|
|
||||||
logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
|
logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
|
||||||
delivery_tag, message.consumer_tag, amq_message.id,
|
message.delivery_tag, message.consumer_tag, amq_message.id,
|
||||||
amq_message.method, amq_message.path, map_as_string(amq_message.trace_info))
|
amq_message.method, amq_message.path, map_as_string(amq_message.trace_info))
|
||||||
|
return amq_message
|
||||||
|
|
||||||
propagator = TraceContextTextMapPropagator()
|
def ensure_trace_info(self, amq_message: DataMessage):
|
||||||
context = propagator.extract(carrier=amq_message.trace_info, context=trace.context_api.get_current())
|
propagator = TraceContextTextMapPropagator()
|
||||||
self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
|
context = propagator.extract(carrier=amq_message.trace_info, context=trace.context_api.get_current())
|
||||||
|
self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
|
||||||
logging.info(" [*] CTX=%s", context)
|
logging.info(" [*] CTX=%s", context)
|
||||||
# current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
|
# current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
|
||||||
# context_with_span = trace.set_span_in_context(current_span, context)
|
# context_with_span = trace.set_span_in_context(current_span, context)
|
||||||
# logging.info(" [*] CTX2=%s", context_with_span)
|
# logging.info(" [*] CTX2=%s", context_with_span)
|
||||||
# propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
|
# propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
|
||||||
# logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
|
# logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
|
||||||
# map_as_string(amq_message.trace_info), context_with_span, context)
|
# map_as_string(amq_message.trace_info), context_with_span, context)
|
||||||
|
|
||||||
if amq_message.magic == DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
|
|
||||||
future = concurrent.futures.ThreadPoolExecutor().submit(
|
|
||||||
self.sync_on_message, message, amq_message, self.loop
|
|
||||||
)
|
|
||||||
# response: DataResponse = asyncio.run(self.http_adapter.on_message(amq_message))
|
|
||||||
# Wait and send it back to the reply queue
|
|
||||||
#try:
|
|
||||||
# response: DataResponse = future.result() # Block until the result is available
|
|
||||||
# await self.send_reply(message.reply_to, delivery_tag, response)
|
|
||||||
#except Exception as e:
|
|
||||||
# logging.info(f"[E] Error processing message: {e}")
|
|
||||||
# tb = traceback.extract_tb(sys.exc_info()[2])[-1]
|
|
||||||
# logging.error(f"[E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
|
|
||||||
# await message.nack(requeue=False)
|
|
||||||
# return
|
|
||||||
else:
|
|
||||||
logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
|
|
||||||
amq_message.magic)
|
|
||||||
await message.nack(requeue=False)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
logging.error(" [E] ######[%s] No valid AMQ Message present.", delivery_tag)
|
|
||||||
|
|
||||||
|
def record_duration(self, start, delivery_tag):
|
||||||
global last_data_message_time
|
global last_data_message_time
|
||||||
last_data_message_time = time.time()
|
last_data_message_time = time.time()
|
||||||
logging.info(" [x] ######[%s] Done, single ACK: %sms.", delivery_tag, last_data_message_time - start)
|
logging.info(" [x] ######[%s] Done, single ACK: %sms.", delivery_tag, last_data_message_time - start)
|
||||||
await message.ack(multiple=False)
|
|
||||||
|
|
||||||
async def send_reply(self, reply_to: str, response: DataResponse):
|
async def send_reply(self, reply_to: str, response: DataResponse):
|
||||||
"""
|
"""
|
||||||
The Service side code. When the CleverXXX service has output ready, in DataResponse object,
|
The Service side code. When the CleverXXX service has output ready, in DataResponse object,
|
||||||
call this method to return the reply back to the caller
|
call this method to return the reply back to the caller
|
||||||
:param reply_to:
|
:param reply_to:connection.channel
|
||||||
:param response:
|
:param response:
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
@@ -306,10 +293,10 @@ class DataMessageHandler:
|
|||||||
if isinstance(response.content_type, list) else response.content_type
|
if isinstance(response.content_type, list) else response.content_type
|
||||||
)
|
)
|
||||||
logging.info(" [*] ###### Sending Reply.1 To=%s, msgId=%s", reply_to, response.id)
|
logging.info(" [*] ###### Sending Reply.1 To=%s, msgId=%s", reply_to, response.id)
|
||||||
_res = await self.reply_to_exchange.publish(
|
_res = asyncio.run_coroutine_threadsafe(self.reply_to_exchange.publish(
|
||||||
message=pika_message,
|
message=pika_message,
|
||||||
routing_key=reply_to
|
routing_key=reply_to
|
||||||
)
|
), self.loop)
|
||||||
logging.info(" [*] ###### DONE EXCH PUB(%s) Sending Reply.2 To=%s, res=%s", self.reply_to_exchange.name, reply_to, _res)
|
logging.info(" [*] ###### DONE EXCH PUB(%s) Sending Reply.2 To=%s, res=%s", self.reply_to_exchange.name, reply_to, _res)
|
||||||
|
|
||||||
async def reply_received_callback(self, message: AbstractIncomingMessage):
|
async def reply_received_callback(self, message: AbstractIncomingMessage):
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ class AMQService:
|
|||||||
self.loop = asyncio.get_event_loop()
|
self.loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
self.amq_configuration = amq_configuration
|
self.amq_configuration = amq_configuration
|
||||||
self.message_factory = DataMessageFactory.get_instance(self.amq_configuration.amq_adapter.generator_id)
|
self.data_message_factory = DataMessageFactory.get_instance(self.amq_configuration.amq_adapter.generator_id)
|
||||||
self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
|
self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
|
||||||
self.tracer = initialize_trace(name=amq_configuration.amq_adapter.service_name)
|
self.tracer = initialize_trace(name=amq_configuration.amq_adapter.service_name)
|
||||||
self.amq_data_message_handler = None
|
self.amq_data_message_handler = None
|
||||||
@@ -47,9 +47,6 @@ class AMQService:
|
|||||||
)
|
)
|
||||||
# self.init(loop) will initialize RabbitMQ connection
|
# self.init(loop) will initialize RabbitMQ connection
|
||||||
self.loop.run_until_complete(self.init(self.loop, service_adapter))
|
self.loop.run_until_complete(self.init(self.loop, service_adapter))
|
||||||
# loop.run_until_complete(consume(loop))
|
|
||||||
# self.consumer_thread = asyncio.create_task(self.rabbit_mq_client.channel.start_consuming())
|
|
||||||
# self.rabbit_mq_client.request_routes()
|
|
||||||
|
|
||||||
logging.info("***********************************************")
|
logging.info("***********************************************")
|
||||||
logging.info("****** AMQ Service Init Complete *********")
|
logging.info("****** AMQ Service Init Complete *********")
|
||||||
@@ -57,13 +54,13 @@ class AMQService:
|
|||||||
self.loop.run_forever()
|
self.loop.run_forever()
|
||||||
|
|
||||||
async def init(self, loop, service_adapter):
|
async def init(self, loop, service_adapter):
|
||||||
exchange: AbstractRobustExchange = await self.rabbit_mq_client.init(loop)
|
_reply_to_exchange: AbstractRobustExchange = await self.rabbit_mq_client.init(loop)
|
||||||
self.amq_data_message_handler = DataMessageHandler(
|
self.amq_data_message_handler = DataMessageHandler(
|
||||||
service_adapter,
|
service_adapter,
|
||||||
self.tracer,
|
self.tracer,
|
||||||
self.message_factory,
|
self.data_message_factory,
|
||||||
self.service_message_factory,
|
self.service_message_factory,
|
||||||
exchange,
|
_reply_to_exchange,
|
||||||
self.rabbit_mq_client,
|
self.rabbit_mq_client,
|
||||||
self.amq_configuration,
|
self.amq_configuration,
|
||||||
loop=loop
|
loop=loop
|
||||||
|
|||||||
+1
-1
@@ -8,7 +8,7 @@ packages = ["amqp","amqp.adapter","amqp.config","amqp.model","amqp.rabbitmq","am
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "amq_adapter"
|
name = "amq_adapter"
|
||||||
version = "0.2.18"
|
version = "0.2.19"
|
||||||
description = "The Python implementation of the AMQ Adaptor for CleverMicro"
|
description = "The Python implementation of the AMQ Adaptor for CleverMicro"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = { text = "Apache License 2.0" }
|
license = { text = "Apache License 2.0" }
|
||||||
|
|||||||
Reference in New Issue
Block a user