Added support for file download

amqp/adapter/cleverthis_service_adapter.py

@@ -2,25 +2,45 @@
 The layer that interfaces with the CleverThis service that this Adapter is integrating with.
 Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
 """
-
+import asyncio
+import inspect
 import json
 import logging
-import re
 import sys
 import traceback
-from re import Pattern
+from asyncio import Future
-from typing import Dict, List, Tuple, Callable, Coroutine, AnyStr, Any
+from typing import Dict, List, Tuple, Callable, Coroutine, Any
 from urllib.parse import parse_qs, urlparse
 from xml.etree import ElementTree
 
+from aio_pika.abc import AbstractRobustChannel
 from aiohttp import ClientSession, ClientResponse
+from dataclasses import dataclass
 from fastapi import HTTPException
+from starlette.responses import FileResponse
 
-from amqp.adapter import pydantic_serializer
 from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.pydantic_serializer import JSONConverter
 from amqp.config.amq_configuration import AMQConfiguration
 from amqp.model.model import DataMessage, DataResponse
-from amqp.service.multipart_parser import CleverMultiPartParser
+from amqp.adapter.multipart_parser import CleverMultiPartParser, process_form_data
+from amqp.adapter.file_uploader import publish_file_to_rabbitmq
+
+
+@dataclass
+class AMQMessage(DataMessage):
+    def __init__(self, data_message: DataMessage, channel: AbstractRobustChannel):
+        # Copy all fields from DataMessage
+        self.magic = data_message.magic
+        self.id = data_message.id
+        self.method = data_message.method
+        self.domain = data_message.domain
+        self.path = data_message.path
+        self.headers = data_message.headers
+        self.trace_info = data_message.trace_info
+        self.base64_body = data_message.base64_body
+        # Add the new field
+        self.channel = channel
 
 
 def parse_request_body(message: DataMessage):
@@ -42,17 +62,22 @@ def parse_request_body(message: DataMessage):
         try:
             return json.loads(message.body())
         except json.JSONDecodeError as e:
-            tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+            _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-            logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
+            logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
             raise ValueError(f"Invalid JSON: {e}")
 
     # Parse URL-encoded form data
     elif content_type == "application/x-www-form-urlencoded":
         try:
-            return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
+            _form_data: dict = {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
+            if len(_form_data) == 0 and len(message.body()) > 0:
+                _form_data = json.loads(message.body())
+                if _form_data['files'] and _form_data['form']:
+                    _form_data = process_form_data(_form_data)
+            return _form_data
         except Exception as e:
-            tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+            _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-            logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
+            logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
             raise ValueError(f"Invalid URL-encoded form data: {e}")
 
     # Parse multipart/form-data
@@ -61,8 +86,8 @@ def parse_request_body(message: DataMessage):
             parser: CleverMultiPartParser = CleverMultiPartParser(message)
             return parser.form_data
         except Exception as e:
-            tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+            _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-            logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {e}")
+            logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {e}")
             raise ValueError(f"Invalid multipart/form-data: {e}")
 
     # Parse plain text
@@ -72,8 +97,8 @@ def parse_request_body(message: DataMessage):
     # Parse XML
     elif content_type == "application/xml":
         try:
-            root = ElementTree.fromstring(message.body())
+            _root = ElementTree.fromstring(message.body())
-            return {elem.tag: elem.text for elem in root}
+            return {elem.tag: elem.text for elem in _root}
         except ElementTree.ParseError as e:
             raise ValueError(f"Invalid XML: {e}")
 
@@ -81,8 +106,22 @@ def parse_request_body(message: DataMessage):
         raise ValueError(f"Unsupported Content-Type: {content_type}")
 
 
-async def call_cleverthis_post_put_api(
+async def convert_file_response(obj: FileResponse, channel: AbstractRobustChannel) -> str:
-        message: DataMessage,
+    """
+    Converts a FileResponse object to a JSON string containing the filename and stream.
+    """
+    _future: Future = asyncio.run_coroutine_threadsafe(
+        publish_file_to_rabbitmq(channel, obj.path), loop=channel.channel.loop)
+    return json.dumps({'files':[
+        {'filename': obj.filename,
+         'mediaType': 'application/octet-stream',
+         'streamName': _future.result()
+         }]
+    })
+
+
+async def call_cleverthis_api(
+        message: AMQMessage,
         args: Any,
         api_method: Callable[[Any, Any], Coroutine],
         success_code: int
@@ -98,19 +137,34 @@ async def call_cleverthis_post_put_api(
     :return: DataResponse class
     """
     try:
-        result: str = await api_method(**args)
+        _verified_args = {}
-        return DataResponseFactory.of(
+        _sig = inspect.signature(api_method)
-            message.id, success_code, message.headers.get("Content-Type", "application/json"),
+        _required_args = list(_sig.parameters.values())
-            result.encode('utf-8'), message.trace_info
+        #_required_args = getattr(api_method, '__annotations__', {})
-        )
+        for arg in _required_args:
+            if arg.name in args:
+                _verified_args[arg.name] = args[arg.name]
+            else:
+                logging.info(f"{api_method.__name__}: Missing required argument: {arg.name}")
+        logging.info(
+            f" [*] INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}")
+        _result: Any = await api_method(**_verified_args)
+        logging.info(f" [*] ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}")
+        _json_result, _content_type = await convert_file_response(_result, message.channel), 'application/octet-stream' \
+            if isinstance(_result, FileResponse) else _result, 'application/json'
+        _binary_result = _json_result.encode('utf-8') if hasattr(_json_result, 'encode') else \
+            _json_result.body if hasattr(_json_result, 'body') else \
+            _json_result.read() if hasattr(_json_result, 'read') else \
+            _json_result
+        return DataResponseFactory.of(message.id, success_code, _content_type, _binary_result, message.trace_info)
     except HTTPException as http_error:
-        tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+        _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-        logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
+        logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {http_error}")
         return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
                                      str(http_error.detail).encode('utf-8'), message.trace_info)
     except Exception as error:
-        tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+        _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-        logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
+        logging.error(f" [E] Error in {_tb.filename}, line {_tb.lineno}, function '{_tb.name}': {error}")
         return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
 
 
@@ -126,61 +180,11 @@ def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
         - context_path: The path part of the URL (e.g., "/path")
         - query_params_dict: Query parameters where values are str for single value or list of str for multivalue
     """
-    parsed = urlparse(url)
+    _parsed = urlparse(url)
-    query_params = parse_qs(parsed.query)
+    _query_params = parse_qs(_parsed.query)
     # Convert values from lists to single values when there's only one value
-    simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
+    _simplified_params = {k: v[0] if len(v) == 1 else v for k, v in _query_params.items()}
-    return parsed.path, simplified_params
+    return _parsed.path, _simplified_params
-
-async def call_cleverthis_get_api(
-        message: DataMessage,
-        pattern_or_data: Pattern| str | dict,
-        api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine],
-        authenticated_user: Any
-) -> DataResponse:
-    """
-    Handles GET requests, which do not contain a body, but may have path variables and extra values
-    in the query string.
-    invokes provided Callable that is expected to be a GET API method, and returns the response as DataResponse
-    :param message: DataMessage containing the details of the call
-    :param pattern_or_data: If there are path variables, use this pattern to extract them
-    :param api_method: Callable - the method to invoke.
-    :param authenticated_user: Authenticated user. The actual class depends on the service
-    :return:
-    """
-    try:
-        args = None
-        if isinstance(pattern_or_data, dict):
-            args = pattern_or_data
-        elif isinstance(pattern_or_data, Pattern):
-            path, args = pydantic_serializer.parse_url_query(message.path)
-            match = pattern_or_data.match(path)
-            if match:
-                args.update(match.groupdict())
-            else:
-                _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
-                return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
-        else:
-            path, args = pydantic_serializer.parse_url_query(message.path)
-            match = re.search(pattern_or_data, path)
-            if match:
-                args.update(match.groupdict())
-            else:
-                _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
-                return DataResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
-        crt: Coroutine = api_method(args, authenticated_user)
-        result: str = await crt
-        return DataResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
-
-    except HTTPException as http_error:
-        tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-        logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")
-        return DataResponseFactory.of(message.id, http_error.status_code, "text/plain",
-                                     str(http_error.detail).encode('utf-8'), message.trace_info)
-    except Exception as error:
-        tb = traceback.extract_tb(sys.exc_info()[2])[-1]
-        logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {error}")
-        return DataResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
 
 
 # =============  CleverThisServiceAdapter =============
@@ -192,6 +196,11 @@ class CleverThisServiceAdapter:
     and override the methods to handle the specific API calls.
     """
     def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
+        """
+        Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session.
+        The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via
+        HTTP REST API.
+        """
         self.amq_configuration = amq_configuration
         self.session = session
         self.service_host = self.amq_configuration.dispatch.service_host
@@ -218,14 +227,14 @@ class CleverThisServiceAdapter:
         """
         return None
 
-    async def on_message(self, message) -> DataResponse:
+    async def on_message(self, message: AMQMessage) -> DataResponse:
         """
         Called when a message is received from the AMQP.
         :param message:
         :return: AMQP response class with operation status code and optional error details.
         """
         _auth_user: Any | None = None
-        amq_response: DataResponse | None = None
+        amq_response: Any | None = None
 
         if self.require_authenticated_user:
             _auth: str = message.headers.get('Authorization', '')
@@ -317,7 +326,7 @@ class CleverThisServiceAdapter:
         )
 
     async def handle_possible_form(self, message: DataMessage, request_coroutine, auth_user: Any) -> DataResponse:
-        args = parse_request_body(message)
+        _args = parse_request_body(message)
         return await request_coroutine
 
     async def head(self, message: DataMessage, auth_user: Any) -> DataResponse:

amqp/adapter/data_message_factory.py

@@ -11,7 +11,7 @@ from typing import Dict, List
 
 from amqp.model.model import DataMessage
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string
+from amqp.adapter.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string
 from amqp.router.utils import sanitize_as_rabbitmq_name
 from amqp.adapter.file_downloader import download_buffer
 

amqp/adapter/data_response_factory.py

@@ -7,7 +7,7 @@ import re
 from io import BytesIO
 from amqp.model.model import DataResponse
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
+from amqp.adapter.serializer import parse_map_string, map_as_string
 
 
 class DataResponseFactory:

amqp/adapter/file_downloader.py

@@ -162,8 +162,8 @@ async def on_message(
                 files_to_download.append(files_info)
 
         if not files_to_download:
-            logging.info("No files to download in this message.")
+            logging.info(" [%s] No files to download in this message.", message.delivery_tag)
-            await message.ack()
+            # await message.ack()
             return False
 
         # Create a task for each file download

amqp/adapter/file_uploader.py

@@ -0,0 +1,73 @@
+import logging
+import aio_pika
+import asyncio
+import os
+
+from pathlib import Path
+from aio_pika.abc import AbstractRobustChannel
+from aio_pika import Message, DeliveryMode
+from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
+
+
+async def publish_file_to_rabbitmq(
+    channel: AbstractRobustChannel,
+    file_path: Path,
+    max_chunk_size: int = 2**20,  # 1MB default chunk size
+    exchange_name: str = AMQ_REPLY_TO_EXCHANGE,
+    content_type: str = 'application/octet-stream',  # Default content type
+    delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+) -> str:
+    """
+    Reads a file from the file system and publishes its content to a RabbitMQ exchange
+    in chunks.  It declares a queue with a random name, binds it to the specified
+    exchange, and uses the queue name as the routing key.
+
+    Args:
+        channel: The aio_pika RobustChannel to use for communication with RabbitMQ.
+        file_path: The path to the file to publish.
+        max_chunk_size: The maximum size of each chunk (in bytes).
+        exchange_name: The name of the RabbitMQ exchange to publish to.
+        content_type: The content type of the file data.
+        delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+    Returns:
+        The name of the randomly generated queue where the file content was published.
+    """
+    if not file_path.is_file():
+        raise FileNotFoundError(f"File not found: {file_path}")
+    _file_size = os.path.getsize(file_path)
+    _exchange = await channel.get_exchange(name=exchange_name, ensure=True)
+    _queue = await channel.declare_queue(exclusive=True)  # Declare a unique, exclusive queue
+    _queue_name = _queue.name
+    await _queue.bind(
+        exchange=exchange_name,
+        routing_key=_queue_name,  # Use queue name as routing key
+    )
+
+    logging.info(f"Publishing file: {file_path} (size: {_file_size}) to exchg: {exchange_name}, q: {_queue_name}")
+
+    with open(file_path, 'rb') as f:
+        _offset = 0
+        while True:
+            _chunk = f.read(max_chunk_size)
+            if not _chunk:
+                break  # End of file
+
+            message = Message(
+                body=_chunk,
+                content_type=content_type,
+                delivery_mode=delivery_mode,
+                headers={
+                    'file_name': file_path.name,
+                    'offset': _offset,
+                    'total_size': _file_size,
+                    'chunk_size': len(_chunk),
+                    'last_chunk': max_chunk_size + _offset >= _file_size
+                }
+            )
+            await _exchange.publish(message, routing_key=_queue_name)
+            _offset += len(_chunk)
+            logging.debug(f"Published chunk {_offset}/{_file_size} to {exchange_name}:{_queue_name}")
+
+    print(f"File {file_path} published to RabbitMQ exchange {exchange_name}, queue {_queue_name}")
+    return _queue_name

amqp/adapter/multipart_parser.py

@@ -10,6 +10,40 @@ from python_multipart.multipart import Field, File
 from amqp.model.model import DataMessage
 
 
+def process_form_data(json_input) -> dict:
+    """
+    Processes the input JSON to extract form data. The JSON has two top entries: 'files' and 'form'.
+    This is part of the file stream implementation in AMQ adapter to stream large files in chunks.
+    'files' contains information about the AMQ queue from which read the file, while 'form' contains
+    additional non-file form values.
+    This function combines the file and values at the top level in the returned dictionary.
+    Args:
+        json_input: JSON string or dict containing the form and files data
+
+    Returns:
+        dict: Processed form data with files data overriding non-empty values
+    """
+    # Load JSON if input is string
+    if isinstance(json_input, str):
+        data = json.loads(json_input)
+    else:
+        data = json_input
+
+    result = {}
+
+    # Process form data - take first element of each list
+    for key, value in data['form'].items():
+        if value:  # Only process if list is not empty
+            result[key] = value[0]
+
+    # Override with files data where available
+    for key, value in data['files'].items():
+        if value:  # Only override if files data is non-empty
+            result[key] = value[0]
+
+    return result
+
+
 class CleverMultiPartParser:
     def __init__(self, message: DataMessage):
         self.form_data = {}
@@ -33,7 +67,7 @@ class CleverMultiPartParser:
                     self.form_data = json.loads(message.body())
                     self.is_json = True
                     if self.form_data['files'] and self.form_data['form']:
-                        self.form_data = self.process_form_data(self.form_data)
+                        self.form_data = process_form_data(self.form_data)
                 except Exception as jsonerror:
                     logging.error(f"Error parsing JSON: {jsonerror}")
                     self.is_valid = False
@@ -50,33 +84,3 @@ class CleverMultiPartParser:
         self.form_data[file.field_name.decode('utf-8')] = \
             UploadFile(file.file_object, size=file.size, filename=file.file_name)
         print(self.form_data)
-
-    def process_form_data(self, json_input):
-        """
-        Processes the input JSON to extract form data, with files data overriding where available.
-
-        Args:
-            json_input: JSON string or dict containing the form and files data
-
-        Returns:
-            dict: Processed form data with files data overriding non-empty values
-        """
-        # Load JSON if input is string
-        if isinstance(json_input, str):
-            data = json.loads(json_input)
-        else:
-            data = json_input
-
-        result = {}
-
-        # Process form data - take first element of each list
-        for key, value in data['form'].items():
-            if value:  # Only process if list is not empty
-                result[key] = value[0]
-
-        # Override with files data where available
-        for key, value in data['files'].items():
-            if value:  # Only override if files data is non-empty
-                result[key] = value[0]
-
-        return result

amqp/adapter/pydantic_serializer.py

@@ -142,29 +142,6 @@ def _is_int_string(s: str) -> bool:
         return False
 
 
-def serialize_to_json(obj: BaseModel) -> str:
-    """
-    Serializes an BaseModel object to a JSON string.  Handles UUIDs and datetimes
-    natively using the default JSON encoder with some help for non-standard types.
-
-    Args:
-        obj: The R2RSerializable object to serialize.
-
-    Returns:
-        A JSON string representation of the object.
-    """
-    def default_serializer(o):
-        if isinstance(o, uuid.UUID):
-            return str(o)
-        elif isinstance(o, datetime):
-            return o.isoformat()
-        elif isinstance(o, BaseModel):
-            return o.model_dump()
-        raise TypeError(f"Object of type {o.__class__.__name__} is not JSON serializable")
-
-    return json.dumps(obj.model_dump(), default=default_serializer)
-
-
 def parse_url_query(url):
     """
     Parses a HTTP GET URL and returns (context_path, query_params_dict)

amqp/adapter/service_message_factory.py

@@ -6,7 +6,7 @@ from amqp.adapter.trace_info_adapter import TraceInfoAdapter
 from amqp.model.model import ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
+from amqp.adapter.serializer import parse_map_string, map_as_string
 
 RS = "|"
 SPLIT_RS = r"\|"

amqp/model/model.py

@@ -1,9 +1,6 @@
 import base64
 import json
 from enum import Enum
-from dataclasses import dataclass
-from typing import Final
-
 from typing import List, Dict
 from dataclasses import dataclass, field
 
@@ -166,10 +163,6 @@ class DataMessage:
         return base64.b64decode(self.base64_body)
 
 
-# Ensure backward compatibility
-AMQMessage = DataMessage
-
-
 class MultipartDataMessage(DataMessage):
     def __init__(self, message: DataMessage, extra_data: dict):
         super().__init__(message.magic, message.id,

amqp/rabbitmq/rabbit_mq_client.py

@@ -79,29 +79,30 @@ class RabbitMQClient:
                                                                                   exclusive=False,
                                                                                   auto_delete=False,
                                                                                   arguments=queue_args)
-                    exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE
+                    _exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE
                     exchange: AbstractRobustExchange = await self.channel.declare_exchange(
-                        name=exchange_name, type=ExchangeType.topic
+                        name=_exchange_name, type=ExchangeType.topic
                     )
                     await queue.bind(exchange, route.key)
-                    await queue.consume(callback, no_ack=False)
+                    _c_tag = await queue.consume(callback, no_ack=False)
                     # Register the route with the router and publish its detail to other adapters
                     await self.router.register_route(
                         AMQRoute(
-                            route.component_name, exchange_name, queue_name,
+                            route.component_name, _exchange_name, queue_name,
                             sanitize_as_rabbitmq_name(route.key),
                             route.timeout, route.valid_until
                         )
                     )
-                    logging.info(" [*] AMQ connection initialized for route: %s, listens on: %s/%s",
+                    logging.info(" [%s] AMQ connection initialized for route: %s, listens on: %s/%s",
-                                 route, exchange, queue_name)
+                                 _c_tag, route, exchange, queue_name)
                 except Exception as err:
                     logging.error(" [E] Callback registration INCOMPLETE, %s", err)
         else:
             logging.warning(" [W] Channel is null, cannot register routes.")
 
     async def register_data_reply_callback(self, rpc_callback):
-        await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False)
+        _c_tag = await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False)
+        logging.info(" [%s] RabbitMQClient register data reply callback on queue: %s", _c_tag,self.reply_to_queue.name)
 
     async def setup_amq_channel(self, loop):
         """

amqp/router/router_consumer.py

@@ -36,13 +36,13 @@ class RouterConsumer(RouterProducer):
         if self.is_open(_channel):
             try:
                 #  1. declare a Queue to receive the RoutingData requests
-                cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
+                _cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
-                queue: AbstractRobustQueue = await self.channel.declare_queue(name=cm_request_queue, durable=True, exclusive=False, auto_delete=False)
+                _queue: AbstractRobustQueue = await self.channel.declare_queue(name=_cm_request_queue, durable=True, exclusive=False, auto_delete=False)
                 # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key)
-                await queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
+                await _queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
                 # 1b. Service adapter mode - listens on Request queue and replies to Response queue
-                logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue)
+                _c_tag = await _queue.consume(callback=self.handle_routing_data_request_message, no_ack=False)
-                await queue.consume(callback=self.handle_routing_data_request_message, no_ack=False)
+                logging.info(" [%s] RouterConsumer listens for RouteDataRequests on: %s", _c_tag, _cm_request_queue)
             except Exception as ioe:
                 logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe)
         else:

amqp/router/router_producer.py

@@ -65,16 +65,16 @@ class RouterProducer(RouterBase):
                     name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout
                 )
                 # 2a. declare a Response queue for Routing Data replies from Services
-                cm_response_queue: str = self.get_response_queue_name()
+                _cm_response_queue: str = self.get_response_queue_name()
-                response_queue: AbstractRobustQueue = await self.channel.declare_queue(
+                _response_queue: AbstractRobustQueue = await self.channel.declare_queue(
-                    name=cm_response_queue, durable=True, exclusive=False, auto_delete=False,
+                    name=_cm_response_queue, durable=True, exclusive=False, auto_delete=False,
                     arguments={}
                 )
                 # 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                await response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
+                await _response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
                 # 2c. listen for incoming RoutingData messages
-                await response_queue.consume(callback=self.handle_routing_data_message, no_ack=False)
+                _c_tag = await _response_queue.consume(callback=self.handle_routing_data_message, no_ack=False)
-                logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
+                logging.info(" [%s] Routing master listens for Route updates on %s", _c_tag, _cm_response_queue)
             except Exception as ioe:
                 logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
 
@@ -92,7 +92,7 @@ class RouterProducer(RouterBase):
               delivery.getEnvelope().getDeliveryTag(),
               delivery.getEnvelope().toString(), str(delivery.getBody()))
               """
-        # ACK message now so that it is not being redelivered
+        logging.info(" [%s] ACK message now so that it is not being redelivered", message.delivery_tag)
         await message.ack(multiple=False)
         cm_message: ServiceMessage = self.service_message_factory.from_bytes(message.body)
         # ignore the message if it comes from ourselves (even if from different instance)

amqp/service/amq_message_handler.py

@@ -18,7 +18,7 @@ from opentelemetry.trace import Tracer, TraceState
 
 from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.adapter.data_response_factory import DataResponseFactory
-from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
 from amqp.adapter.file_downloader import on_message
 from amqp.adapter.service_message_factory import ServiceMessageFactory
 from amqp.config.amq_configuration import AMQConfiguration
@@ -26,7 +26,7 @@ from amqp.model.model import DataResponse, DataMessage, MultipartDataMessage, Sc
 from amqp.model.service_message_type import ServiceMessageType
 from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
 from amqp.router.router_base import ANY_RECIPIENT
-from amqp.router.serializer import map_as_string
+from amqp.adapter.serializer import map_as_string
 
 # Track the number of messages currently being processed
 current_parallel_executions = 0
@@ -178,11 +178,15 @@ class DataMessageHandler:
                 last_data_message_time = time.time()
 
         # Call the async on_message method. This shall also ACK the message after processing.
-        response: DataResponse = asyncio.run(self.service_adapter.on_message(amq_message))
+        response: DataResponse = asyncio.run(
+            self.service_adapter.on_message(AMQMessage(amq_message, self.rabbit_mq_client.channel))
+        )
         try:
-            logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s",
+            logging.info(" [*] ######[%s] AMQ Message Reply.1, To=%s, msgId=%s",
                          message.delivery_tag, message.reply_to, response.id)
-            asyncio.run_coroutine_threadsafe(self.send_reply(message.reply_to, response), loop=loop)
+            future: Future = asyncio.run_coroutine_threadsafe(self.send_reply(message.reply_to, response), loop=loop)
+            logging.info(" [*] ######[%s] AMQ Message Reply.2, To=%s, future=%s, done=%s",
+                         message.delivery_tag, message.reply_to, future, future.done())
         except Exception as e:
             logging.info(f"[E] Error processing message: {e}")
             tb = traceback.extract_tb(sys.exc_info()[2])[-1]
@@ -236,8 +240,9 @@ class DataMessageHandler:
                 amq_message = MultipartDataMessage(amq_message, extra_data)
 
         if amq_message is not None:
-            logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s",
+            logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
-                         delivery_tag, message.consumer_tag, amq_message.id, map_as_string(amq_message.trace_info))
+                         delivery_tag, message.consumer_tag, amq_message.id,
+                         amq_message.method, amq_message.path, map_as_string(amq_message.trace_info))
 
             propagator = TraceContextTextMapPropagator()
             context = propagator.extract(carrier=amq_message.trace_info, context=trace.context_api.get_current())
@@ -292,10 +297,12 @@ class DataMessageHandler:
                 content_type=response.content_type[0]
                 if isinstance(response.content_type, list) else response.content_type
             )
-            await self.reply_to_exchange.publish(
+            logging.info(" [*] ###### Sending Reply.1 To=%s, msgId=%s", reply_to, response.id)
+            _res = await self.reply_to_exchange.publish(
                 message=pika_message,
                 routing_key=reply_to
             )
+            logging.info(" [*] ###### Sending Reply.2 To=%s, res=%s", reply_to, _res)
 
     async def reply_received_callback(self, message: AbstractIncomingMessage):
         """

cleverswarm/clever_swarm_adapter.py

@@ -20,18 +20,18 @@ from schemas.user_schema import UserSchema
 
 # AMQP specific imports
 from amqp.adapter.data_response_factory import AMQResponseFactory
-from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, call_cleverthis_get_api, \
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, \
-    call_cleverthis_post_put_api, parse_get_url
+    parse_get_url, call_cleverthis_api, AMQMessage
-from amqp.service.multipart_parser import CleverMultiPartParser
+from amqp.adapter.multipart_parser import CleverMultiPartParser
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage, AMQResponse
+from amqp.model.model import AMQResponse
 from amqp.service.amq_service import AMQService
 
 
-
 # Suffix for the method name with serialization wrapper, prefer use this one over the orig endpoint
 PREFERRED_SUFFIX = '_json'
 
+
 def clone_and_adapt_route(original_route: APIRoute) -> APIRoute:
     """
     Creates a clone of an APIRoute. It checks if there's preferred endpoint with json serialization.
@@ -43,7 +43,8 @@ def clone_and_adapt_route(original_route: APIRoute) -> APIRoute:
         getattr(original_route.endpoint, '__module__'), original_route.endpoint.__name__ + PREFERRED_SUFFIX
     )
     # Need to adapt also the path regex to ensure it matches the path in different API versions
-    _new_regex_string = '^.*' + original_route.path_regex.pattern.lstrip('^')  # Ensures '^' is only at the start
+    # Ensures the leading '^' is followed by any match and there's no '/' before final '$'
+    _new_regex_string = '^.*' + original_route.path_regex.pattern.lstrip('^').rstrip('/$') +'$'
     _new_route: APIRoute = APIRoute(
         path=original_route.path,
         endpoint=_preferred_endpoint if _preferred_endpoint else original_route.endpoint,
@@ -189,7 +190,7 @@ class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
                 args.update(match.groupdict())
                 args['authenticated_user'] = auth_user
                 # If the path matches, call the corresponding function
-                return await call_cleverthis_post_put_api(
+                return await call_cleverthis_api(
                     message,
                     args,
                     api_method=route.endpoint,
@@ -231,7 +232,7 @@ class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
                             ) or parser.form_data[body_param.name]
 
 
-                return await call_cleverthis_post_put_api(
+                return await call_cleverthis_api(
                     message,
                     parser.form_data,
                     api_method=route.endpoint,

pyproject.toml

@@ -8,7 +8,7 @@ packages = ["amqp","amqp.adapter","amqp.config","amqp.model","amqp.rabbitmq","am
 
 [project]
 name = "amq_adapter"
-version = "0.2.14"
+version = "0.2.16"
 description = "The Python implementation of the AMQ Adaptor for CleverMicro"
 readme = "README.md"
 license = { text = "Apache License 2.0" }