diff --git a/amqp/adapter/cleverthis_service_adapter.py b/amqp/adapter/cleverthis_service_adapter.py index 88e0755..9eebb88 100644 --- a/amqp/adapter/cleverthis_service_adapter.py +++ b/amqp/adapter/cleverthis_service_adapter.py @@ -2,25 +2,45 @@ The layer that interfaces with the CleverThis service that this Adapter is integrating with. Methods here will invoke the actual CleverThis Service code, which are passed as Callable object. """ - +import asyncio +import inspect import json import logging -import re import sys import traceback -from re import Pattern -from typing import Dict, List, Tuple, Callable, Coroutine, AnyStr, Any +from asyncio import Future +from typing import Dict, List, Tuple, Callable, Coroutine, Any from urllib.parse import parse_qs, urlparse from xml.etree import ElementTree +from aio_pika.abc import AbstractRobustChannel from aiohttp import ClientSession, ClientResponse +from dataclasses import dataclass from fastapi import HTTPException +from starlette.responses import FileResponse -from amqp.adapter import pydantic_serializer from amqp.adapter.data_response_factory import DataResponseFactory +from amqp.adapter.pydantic_serializer import JSONConverter from amqp.config.amq_configuration import AMQConfiguration from amqp.model.model import DataMessage, DataResponse -from amqp.service.multipart_parser import CleverMultiPartParser +from amqp.adapter.multipart_parser import CleverMultiPartParser, process_form_data +from amqp.adapter.file_uploader import publish_file_to_rabbitmq + + +@dataclass +class AMQMessage(DataMessage): + def __init__(self, data_message: DataMessage, channel: AbstractRobustChannel): + # Copy all fields from DataMessage + self.magic = data_message.magic + self.id = data_message.id + self.method = data_message.method + self.domain = data_message.domain + self.path = data_message.path + self.headers = data_message.headers + self.trace_info = data_message.trace_info + self.base64_body = data_message.base64_body + # Add the new field + self.channel = channel def parse_request_body(message: DataMessage): @@ -42,17 +62,22 @@ def parse_request_body(message: DataMessage): try: return json.loads(message.body()) except json.JSONDecodeError as e: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}") + _tb = traceback.extract_tb(sys.exc_info()[2])[-1] + logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}") raise ValueError(f"Invalid JSON: {e}") # Parse URL-encoded form data elif content_type == "application/x-www-form-urlencoded": try: - return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()} + _form_data: dict = {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()} + if len(_form_data) == 0 and len(message.body()) > 0: + _form_data = json.loads(message.body()) + if _form_data['files'] and _form_data['form']: + _form_data = process_form_data(_form_data) + return _form_data except Exception as e: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}") + _tb = traceback.extract_tb(sys.exc_info()[2])[-1] + logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}") raise ValueError(f"Invalid URL-encoded form data: {e}") # Parse multipart/form-data @@ -61,8 +86,8 @@ def parse_request_body(message: DataMessage): parser: CleverMultiPartParser = CleverMultiPartParser(message) return parser.form_data except Exception as e: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}") + _tb = traceback.extract_tb(sys.exc_info()[2])[-1] + logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}") raise ValueError(f"Invalid multipart/form-data: {e}") # Parse plain text @@ -72,8 +97,8 @@ def parse_request_body(message: DataMessage): # Parse XML elif content_type == "application/xml": try: - root = ElementTree.fromstring(message.body()) - return {elem.tag: elem.text for elem in root} + _root = ElementTree.fromstring(message.body()) + return {elem.tag: elem.text for elem in _root} except ElementTree.ParseError as e: raise ValueError(f"Invalid XML: {e}") @@ -81,8 +106,22 @@ def parse_request_body(message: DataMessage): raise ValueError(f"Unsupported Content-Type: {content_type}") -async def call_cleverthis_post_put_api( - message: DataMessage, +async def convert_file_response(obj: FileResponse, channel: AbstractRobustChannel) -> str: + """ + Converts a FileResponse object to a JSON string containing the filename and stream. + """ + _future: Future = asyncio.run_coroutine_threadsafe( + publish_file_to_rabbitmq(channel, obj.path), loop=channel.channel.loop) + return json.dumps({'files':[ + {'filename': obj.filename, + 'mediaType': 'application/octet-stream', + 'streamName': _future.result() + }] + }) + + +async def call_cleverthis_api( + message: AMQMessage, args: Any, api_method: Callable[[Any, Any], Coroutine], success_code: int @@ -98,19 +137,34 @@ async def call_cleverthis_post_put_api( :return: DataResponse class """ try: - result: str = await api_method(**args) - return DataResponseFactory.of( - message.id, success_code, message.headers.get("Content-Type", "application/json"), - result.encode('utf-8'), message.trace_info - ) + _verified_args = {} + _sig = inspect.signature(api_method) + _required_args = list(_sig.parameters.values()) + #_required_args = getattr(api_method, '__annotations__', {}) + for arg in _required_args: + if arg.name in args: + _verified_args[arg.name] = args[arg.name] + else: + logging.info(f"{api_method.__name__}: Missing required argument: {arg.name}") + logging.info( + f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}") + _result: Any = await api_method(**_verified_args) + logging.info(f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}") + _json_result, _content_type = await convert_file_response(_result, message.channel), 'application/octet-stream' \ + if isinstance(_result, FileResponse) else _result, 'application/json' + _binary_result = _json_result.encode('utf-8') if hasattr(_json_result, 'encode') else \ + _json_result.body if hasattr(_json_result, 'body') else \ + _json_result.read() if hasattr(_json_result, 'read') else \ + _json_result + return DataResponseFactory.of(message.id, success_code, _content_type, _binary_result, message.trace_info) except HTTPException as http_error: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}") + _tb = traceback.extract_tb(sys.exc_info()[2])[-1] + logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {http_error}") return DataResponseFactory.of(message.id, http_error.status_code, "text/plain", str(http_error.detail).encode('utf-8'), message.trace_info) except Exception as error: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}") + _tb = traceback.extract_tb(sys.exc_info()[2])[-1] + logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {error}") return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info) @@ -126,61 +180,11 @@ def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]: - context_path: The path part of the URL (e.g., "/path") - query_params_dict: Query parameters where values are str for single value or list of str for multivalue """ - parsed = urlparse(url) - query_params = parse_qs(parsed.query) + _parsed = urlparse(url) + _query_params = parse_qs(_parsed.query) # Convert values from lists to single values when there's only one value - simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()} - return parsed.path, simplified_params - -async def call_cleverthis_get_api( - message: DataMessage, - pattern_or_data: Pattern| str | dict, - api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine], - authenticated_user: Any -) -> DataResponse: - """ - Handles GET requests, which do not contain a body, but may have path variables and extra values - in the query string. - invokes provided Callable that is expected to be a GET API method, and returns the response as DataResponse - :param message: DataMessage containing the details of the call - :param pattern_or_data: If there are path variables, use this pattern to extract them - :param api_method: Callable - the method to invoke. - :param authenticated_user: Authenticated user. The actual class depends on the service - :return: - """ - try: - args = None - if isinstance(pattern_or_data, dict): - args = pattern_or_data - elif isinstance(pattern_or_data, Pattern): - path, args = pydantic_serializer.parse_url_query(message.path) - match = pattern_or_data.match(path) - if match: - args.update(match.groupdict()) - else: - _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}" - return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info) - else: - path, args = pydantic_serializer.parse_url_query(message.path) - match = re.search(pattern_or_data, path) - if match: - args.update(match.groupdict()) - else: - _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}" - return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info) - crt: Coroutine = api_method(args, authenticated_user) - result: str = await crt - return DataResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info) - - except HTTPException as http_error: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}") - return DataResponseFactory.of(message.id, http_error.status_code, "text/plain", - str(http_error.detail).encode('utf-8'), message.trace_info) - except Exception as error: - tb = traceback.extract_tb(sys.exc_info()[2])[-1] - logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}") - return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info) + _simplified_params = {k: v[0] if len(v) == 1 else v for k, v in _query_params.items()} + return _parsed.path, _simplified_params # ============= CleverThisServiceAdapter ============= @@ -192,6 +196,11 @@ class CleverThisServiceAdapter: and override the methods to handle the specific API calls. """ def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None): + """ + Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session. + The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via + HTTP REST API. + """ self.amq_configuration = amq_configuration self.session = session self.service_host = self.amq_configuration.dispatch.service_host @@ -218,14 +227,14 @@ class CleverThisServiceAdapter: """ return None - async def on_message(self, message) -> DataResponse: + async def on_message(self, message: AMQMessage) -> DataResponse: """ Called when a message is received from the AMQP. :param message: :return: AMQP response class with operation status code and optional error details. """ _auth_user: Any | None = None - amq_response: DataResponse | None = None + amq_response: Any | None = None if self.require_authenticated_user: _auth: str = message.headers.get('Authorization', '') @@ -317,7 +326,7 @@ class CleverThisServiceAdapter: ) async def handle_possible_form(self, message: DataMessage, request_coroutine, auth_user: Any) -> DataResponse: - args = parse_request_body(message) + _args = parse_request_body(message) return await request_coroutine async def head(self, message: DataMessage, auth_user: Any) -> DataResponse: diff --git a/amqp/adapter/data_message_factory.py b/amqp/adapter/data_message_factory.py index b141a23..0e03f6c 100644 --- a/amqp/adapter/data_message_factory.py +++ b/amqp/adapter/data_message_factory.py @@ -11,7 +11,7 @@ from typing import Dict, List from amqp.model.model import DataMessage from amqp.model.snowflake_id import SnowflakeId -from amqp.router.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string +from amqp.adapter.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string from amqp.router.utils import sanitize_as_rabbitmq_name from amqp.adapter.file_downloader import download_buffer diff --git a/amqp/adapter/data_response_factory.py b/amqp/adapter/data_response_factory.py index 60483dd..1c29aaa 100644 --- a/amqp/adapter/data_response_factory.py +++ b/amqp/adapter/data_response_factory.py @@ -7,7 +7,7 @@ import re from io import BytesIO from amqp.model.model import DataResponse from amqp.model.snowflake_id import SnowflakeId -from amqp.router.serializer import parse_map_string, map_as_string +from amqp.adapter.serializer import parse_map_string, map_as_string class DataResponseFactory: diff --git a/amqp/adapter/file_downloader.py b/amqp/adapter/file_downloader.py index f37f66a..5d0189e 100644 --- a/amqp/adapter/file_downloader.py +++ b/amqp/adapter/file_downloader.py @@ -162,8 +162,8 @@ async def on_message( files_to_download.append(files_info) if not files_to_download: - logging.info("No files to download in this message.") - await message.ack() + logging.info(" [%s] No files to download in this message.", message.delivery_tag) + # await message.ack() return False # Create a task for each file download diff --git a/amqp/adapter/file_uploader.py b/amqp/adapter/file_uploader.py new file mode 100644 index 0000000..b21694f --- /dev/null +++ b/amqp/adapter/file_uploader.py @@ -0,0 +1,73 @@ +import logging +import aio_pika +import asyncio +import os + +from pathlib import Path +from aio_pika.abc import AbstractRobustChannel +from aio_pika import Message, DeliveryMode +from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE + + +async def publish_file_to_rabbitmq( + channel: AbstractRobustChannel, + file_path: Path, + max_chunk_size: int = 2**20, # 1MB default chunk size + exchange_name: str = AMQ_REPLY_TO_EXCHANGE, + content_type: str = 'application/octet-stream', # Default content type + delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT, +) -> str: + """ + Reads a file from the file system and publishes its content to a RabbitMQ exchange + in chunks. It declares a queue with a random name, binds it to the specified + exchange, and uses the queue name as the routing key. + + Args: + channel: The aio_pika RobustChannel to use for communication with RabbitMQ. + file_path: The path to the file to publish. + max_chunk_size: The maximum size of each chunk (in bytes). + exchange_name: The name of the RabbitMQ exchange to publish to. + content_type: The content type of the file data. + delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT). + + Returns: + The name of the randomly generated queue where the file content was published. + """ + if not file_path.is_file(): + raise FileNotFoundError(f"File not found: {file_path}") + _file_size = os.path.getsize(file_path) + _exchange = await channel.get_exchange(name=exchange_name, ensure=True) + _queue = await channel.declare_queue(exclusive=True) # Declare a unique, exclusive queue + _queue_name = _queue.name + await _queue.bind( + exchange=exchange_name, + routing_key=_queue_name, # Use queue name as routing key + ) + + logging.info(f"Publishing file: {file_path} (size: {_file_size}) to exchg: {exchange_name}, q: {_queue_name}") + + with open(file_path, 'rb') as f: + _offset = 0 + while True: + _chunk = f.read(max_chunk_size) + if not _chunk: + break # End of file + + message = Message( + body=_chunk, + content_type=content_type, + delivery_mode=delivery_mode, + headers={ + 'file_name': file_path.name, + 'offset': _offset, + 'total_size': _file_size, + 'chunk_size': len(_chunk), + 'last_chunk': max_chunk_size + _offset >= _file_size + } + ) + await _exchange.publish(message, routing_key=_queue_name) + _offset += len(_chunk) + logging.debug(f"Published chunk {_offset}/{_file_size} to {exchange_name}:{_queue_name}") + + print(f"File {file_path} published to RabbitMQ exchange {exchange_name}, queue {_queue_name}") + return _queue_name diff --git a/amqp/service/multipart_parser.py b/amqp/adapter/multipart_parser.py similarity index 61% rename from amqp/service/multipart_parser.py rename to amqp/adapter/multipart_parser.py index 489c292..6531aa3 100644 --- a/amqp/service/multipart_parser.py +++ b/amqp/adapter/multipart_parser.py @@ -10,6 +10,40 @@ from python_multipart.multipart import Field, File from amqp.model.model import DataMessage +def process_form_data(json_input) -> dict: + """ + Processes the input JSON to extract form data. The JSON has two top entries: 'files' and 'form'. + This is part of the file stream implementation in AMQ adapter to stream large files in chunks. + 'files' contains information about the AMQ queue from which read the file, while 'form' contains + additional non-file form values. + This function combines the file and values at the top level in the returned dictionary. + Args: + json_input: JSON string or dict containing the form and files data + + Returns: + dict: Processed form data with files data overriding non-empty values + """ + # Load JSON if input is string + if isinstance(json_input, str): + data = json.loads(json_input) + else: + data = json_input + + result = {} + + # Process form data - take first element of each list + for key, value in data['form'].items(): + if value: # Only process if list is not empty + result[key] = value[0] + + # Override with files data where available + for key, value in data['files'].items(): + if value: # Only override if files data is non-empty + result[key] = value[0] + + return result + + class CleverMultiPartParser: def __init__(self, message: DataMessage): self.form_data = {} @@ -33,7 +67,7 @@ class CleverMultiPartParser: self.form_data = json.loads(message.body()) self.is_json = True if self.form_data['files'] and self.form_data['form']: - self.form_data = self.process_form_data(self.form_data) + self.form_data = process_form_data(self.form_data) except Exception as jsonerror: logging.error(f"Error parsing JSON: {jsonerror}") self.is_valid = False @@ -50,33 +84,3 @@ class CleverMultiPartParser: self.form_data[file.field_name.decode('utf-8')] = \ UploadFile(file.file_object, size=file.size, filename=file.file_name) print(self.form_data) - - def process_form_data(self, json_input): - """ - Processes the input JSON to extract form data, with files data overriding where available. - - Args: - json_input: JSON string or dict containing the form and files data - - Returns: - dict: Processed form data with files data overriding non-empty values - """ - # Load JSON if input is string - if isinstance(json_input, str): - data = json.loads(json_input) - else: - data = json_input - - result = {} - - # Process form data - take first element of each list - for key, value in data['form'].items(): - if value: # Only process if list is not empty - result[key] = value[0] - - # Override with files data where available - for key, value in data['files'].items(): - if value: # Only override if files data is non-empty - result[key] = value[0] - - return result diff --git a/amqp/adapter/pydantic_serializer.py b/amqp/adapter/pydantic_serializer.py index 0c19967..6d2caf0 100644 --- a/amqp/adapter/pydantic_serializer.py +++ b/amqp/adapter/pydantic_serializer.py @@ -142,29 +142,6 @@ def _is_int_string(s: str) -> bool: return False -def serialize_to_json(obj: BaseModel) -> str: - """ - Serializes an BaseModel object to a JSON string. Handles UUIDs and datetimes - natively using the default JSON encoder with some help for non-standard types. - - Args: - obj: The R2RSerializable object to serialize. - - Returns: - A JSON string representation of the object. - """ - def default_serializer(o): - if isinstance(o, uuid.UUID): - return str(o) - elif isinstance(o, datetime): - return o.isoformat() - elif isinstance(o, BaseModel): - return o.model_dump() - raise TypeError(f"Object of type {o.__class__.__name__} is not JSON serializable") - - return json.dumps(obj.model_dump(), default=default_serializer) - - def parse_url_query(url): """ Parses a HTTP GET URL and returns (context_path, query_params_dict) diff --git a/amqp/router/serializer.py b/amqp/adapter/serializer.py similarity index 100% rename from amqp/router/serializer.py rename to amqp/adapter/serializer.py diff --git a/amqp/adapter/service_message_factory.py b/amqp/adapter/service_message_factory.py index ef78102..932fe37 100644 --- a/amqp/adapter/service_message_factory.py +++ b/amqp/adapter/service_message_factory.py @@ -6,7 +6,7 @@ from amqp.adapter.trace_info_adapter import TraceInfoAdapter from amqp.model.model import ServiceMessage from amqp.model.service_message_type import ServiceMessageType from amqp.model.snowflake_id import SnowflakeId -from amqp.router.serializer import parse_map_string, map_as_string +from amqp.adapter.serializer import parse_map_string, map_as_string RS = "|" SPLIT_RS = r"\|" diff --git a/amqp/model/model.py b/amqp/model/model.py index 5743c58..6fddd69 100644 --- a/amqp/model/model.py +++ b/amqp/model/model.py @@ -1,9 +1,6 @@ import base64 import json from enum import Enum -from dataclasses import dataclass -from typing import Final - from typing import List, Dict from dataclasses import dataclass, field @@ -166,10 +163,6 @@ class DataMessage: return base64.b64decode(self.base64_body) -# Ensure backward compatibility -AMQMessage = DataMessage - - class MultipartDataMessage(DataMessage): def __init__(self, message: DataMessage, extra_data: dict): super().__init__(message.magic, message.id, diff --git a/amqp/rabbitmq/rabbit_mq_client.py b/amqp/rabbitmq/rabbit_mq_client.py index 0e1a0ea..63f5996 100644 --- a/amqp/rabbitmq/rabbit_mq_client.py +++ b/amqp/rabbitmq/rabbit_mq_client.py @@ -79,29 +79,30 @@ class RabbitMQClient: exclusive=False, auto_delete=False, arguments=queue_args) - exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE + _exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE exchange: AbstractRobustExchange = await self.channel.declare_exchange( - name=exchange_name, type=ExchangeType.topic + name=_exchange_name, type=ExchangeType.topic ) await queue.bind(exchange, route.key) - await queue.consume(callback, no_ack=False) + _c_tag = await queue.consume(callback, no_ack=False) # Register the route with the router and publish its detail to other adapters await self.router.register_route( AMQRoute( - route.component_name, exchange_name, queue_name, + route.component_name, _exchange_name, queue_name, sanitize_as_rabbitmq_name(route.key), route.timeout, route.valid_until ) ) - logging.info(" [*] AMQ connection initialized for route: %s, listens on: %s/%s", - route, exchange, queue_name) + logging.info(" [%s] AMQ connection initialized for route: %s, listens on: %s/%s", + _c_tag, route, exchange, queue_name) except Exception as err: logging.error(" [E] Callback registration INCOMPLETE, %s", err) else: logging.warning(" [W] Channel is null, cannot register routes.") async def register_data_reply_callback(self, rpc_callback): - await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False) + _c_tag = await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False) + logging.info(" [%s] RabbitMQClient register data reply callback on queue: %s", _c_tag,self.reply_to_queue.name) async def setup_amq_channel(self, loop): """ diff --git a/amqp/router/router_consumer.py b/amqp/router/router_consumer.py index 648476f..94d2367 100644 --- a/amqp/router/router_consumer.py +++ b/amqp/router/router_consumer.py @@ -36,13 +36,13 @@ class RouterConsumer(RouterProducer): if self.is_open(_channel): try: # 1. declare a Queue to receive the RoutingData requests - cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE - queue: AbstractRobustQueue = await self.channel.declare_queue(name=cm_request_queue, durable=True, exclusive=False, auto_delete=False) + _cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE + _queue: AbstractRobustQueue = await self.channel.declare_queue(name=_cm_request_queue, durable=True, exclusive=False, auto_delete=False) # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key) - await queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="") + await _queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="") # 1b. Service adapter mode - listens on Request queue and replies to Response queue - logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue) - await queue.consume(callback=self.handle_routing_data_request_message, no_ack=False) + _c_tag = await _queue.consume(callback=self.handle_routing_data_request_message, no_ack=False) + logging.info(" [%s] RouterConsumer listens for RouteDataRequests on: %s", _c_tag, _cm_request_queue) except Exception as ioe: logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe) else: diff --git a/amqp/router/router_producer.py b/amqp/router/router_producer.py index dff385c..7449612 100644 --- a/amqp/router/router_producer.py +++ b/amqp/router/router_producer.py @@ -65,16 +65,16 @@ class RouterProducer(RouterBase): name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout ) # 2a. declare a Response queue for Routing Data replies from Services - cm_response_queue: str = self.get_response_queue_name() - response_queue: AbstractRobustQueue = await self.channel.declare_queue( - name=cm_response_queue, durable=True, exclusive=False, auto_delete=False, + _cm_response_queue: str = self.get_response_queue_name() + _response_queue: AbstractRobustQueue = await self.channel.declare_queue( + name=_cm_response_queue, durable=True, exclusive=False, auto_delete=False, arguments={} ) # 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key) - await response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#") + await _response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#") # 2c. listen for incoming RoutingData messages - await response_queue.consume(callback=self.handle_routing_data_message, no_ack=False) - logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue) + _c_tag = await _response_queue.consume(callback=self.handle_routing_data_message, no_ack=False) + logging.info(" [%s] Routing master listens for Route updates on %s", _c_tag, _cm_response_queue) except Exception as ioe: logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe) @@ -92,7 +92,7 @@ class RouterProducer(RouterBase): delivery.getEnvelope().getDeliveryTag(), delivery.getEnvelope().toString(), str(delivery.getBody())) """ - # ACK message now so that it is not being redelivered + logging.info(" [%s] ACK message now so that it is not being redelivered", message.delivery_tag) await message.ack(multiple=False) cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body) # ignore the message if it comes from ourselves (even if from different instance) diff --git a/amqp/service/amq_message_handler.py b/amqp/service/amq_message_handler.py index cf9ac6d..1a97787 100644 --- a/amqp/service/amq_message_handler.py +++ b/amqp/service/amq_message_handler.py @@ -18,7 +18,7 @@ from opentelemetry.trace import Tracer, TraceState from amqp.adapter.data_message_factory import DataMessageFactory from amqp.adapter.data_response_factory import DataResponseFactory -from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter +from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage from amqp.adapter.file_downloader import on_message from amqp.adapter.service_message_factory import ServiceMessageFactory from amqp.config.amq_configuration import AMQConfiguration @@ -26,7 +26,7 @@ from amqp.model.model import DataResponse, DataMessage, MultipartDataMessage, Sc from amqp.model.service_message_type import ServiceMessageType from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient from amqp.router.router_base import ANY_RECIPIENT -from amqp.router.serializer import map_as_string +from amqp.adapter.serializer import map_as_string # Track the number of messages currently being processed current_parallel_executions = 0 @@ -178,11 +178,15 @@ class DataMessageHandler: last_data_message_time = time.time() # Call the async on_message method. This shall also ACK the message after processing. - response: DataResponse = asyncio.run(self.service_adapter.on_message(amq_message)) + response: DataResponse = asyncio.run( + self.service_adapter.on_message(AMQMessage(amq_message, self.rabbit_mq_client.channel)) + ) try: - logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s", + logging.info(" [*] ######[%s] AMQ Message Reply.1, To=%s, msgId=%s", message.delivery_tag, message.reply_to, response.id) - asyncio.run_coroutine_threadsafe(self.send_reply(message.reply_to, response), loop=loop) + future: Future = asyncio.run_coroutine_threadsafe(self.send_reply(message.reply_to, response), loop=loop) + logging.info(" [*] ######[%s] AMQ Message Reply.2, To=%s, future=%s, done=%s", + message.delivery_tag, message.reply_to, future, future.done()) except Exception as e: logging.info(f"[E] Error processing message: {e}") tb = traceback.extract_tb(sys.exc_info()[2])[-1] @@ -236,8 +240,9 @@ class DataMessageHandler: amq_message = MultipartDataMessage(amq_message, extra_data) if amq_message is not None: - logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s", - delivery_tag, message.consumer_tag, amq_message.id, map_as_string(amq_message.trace_info)) + logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s", + delivery_tag, message.consumer_tag, amq_message.id, + amq_message.method, amq_message.path, map_as_string(amq_message.trace_info)) propagator = TraceContextTextMapPropagator() context = propagator.extract(carrier=amq_message.trace_info, context=trace.context_api.get_current()) @@ -292,10 +297,12 @@ class DataMessageHandler: content_type=response.content_type[0] if isinstance(response.content_type, list) else response.content_type ) - await self.reply_to_exchange.publish( + logging.info(" [*] ###### Sending Reply.1 To=%s, msgId=%s", reply_to, response.id) + _res = await self.reply_to_exchange.publish( message=pika_message, routing_key=reply_to ) + logging.info(" [*] ###### Sending Reply.2 To=%s, res=%s", reply_to, _res) async def reply_received_callback(self, message: AbstractIncomingMessage): """ diff --git a/cleverswarm/clever_swarm_adapter.py b/cleverswarm/clever_swarm_adapter.py index 333501d..49e7e6b 100644 --- a/cleverswarm/clever_swarm_adapter.py +++ b/cleverswarm/clever_swarm_adapter.py @@ -20,18 +20,18 @@ from schemas.user_schema import UserSchema # AMQP specific imports from amqp.adapter.data_response_factory import AMQResponseFactory -from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, call_cleverthis_get_api, \ - call_cleverthis_post_put_api, parse_get_url -from amqp.service.multipart_parser import CleverMultiPartParser +from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, \ + parse_get_url, call_cleverthis_api, AMQMessage +from amqp.adapter.multipart_parser import CleverMultiPartParser from amqp.config.amq_configuration import AMQConfiguration -from amqp.model.model import AMQMessage, AMQResponse +from amqp.model.model import AMQResponse from amqp.service.amq_service import AMQService - # Suffix for the method name with serialization wrapper, prefer use this one over the orig endpoint PREFERRED_SUFFIX = '_json' + def clone_and_adapt_route(original_route: APIRoute) -> APIRoute: """ Creates a clone of an APIRoute. It checks if there's preferred endpoint with json serialization. @@ -43,7 +43,8 @@ def clone_and_adapt_route(original_route: APIRoute) -> APIRoute: getattr(original_route.endpoint, '__module__'), original_route.endpoint.__name__ + PREFERRED_SUFFIX ) # Need to adapt also the path regex to ensure it matches the path in different API versions - _new_regex_string = '^.*' + original_route.path_regex.pattern.lstrip('^') # Ensures '^' is only at the start + # Ensures the leading '^' is followed by any match and there's no '/' before final '$' + _new_regex_string = '^.*' + original_route.path_regex.pattern.lstrip('^').rstrip('/$') +'$' _new_route: APIRoute = APIRoute( path=original_route.path, endpoint=_preferred_endpoint if _preferred_endpoint else original_route.endpoint, @@ -189,7 +190,7 @@ class CleverSwarmAmqpAdapter(CleverThisServiceAdapter): args.update(match.groupdict()) args['authenticated_user'] = auth_user # If the path matches, call the corresponding function - return await call_cleverthis_post_put_api( + return await call_cleverthis_api( message, args, api_method=route.endpoint, @@ -231,7 +232,7 @@ class CleverSwarmAmqpAdapter(CleverThisServiceAdapter): ) or parser.form_data[body_param.name] - return await call_cleverthis_post_put_api( + return await call_cleverthis_api( message, parser.form_data, api_method=route.endpoint, diff --git a/pyproject.toml b/pyproject.toml index f4e63e9..f482958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ packages = ["amqp","amqp.adapter","amqp.config","amqp.model","amqp.rabbitmq","am [project] name = "amq_adapter" -version = "0.2.14" +version = "0.2.16" description = "The Python implementation of the AMQ Adaptor for CleverMicro" readme = "README.md" license = { text = "Apache License 2.0" }