Version that runs with new file upload stream and cleverswarm

README.md

@@ -33,10 +33,26 @@ Then:
 
 ```python
 from amqp.service import amq_service
+```
+
+#### How to build distributable Wheel archive
+
+install project requirements
+`pip install -r requirements.txt`
+
+install distribution archive requirements 
+`pip install setuptools wheel build`
+
+update version in:
+`pyproject.toml`
+
+build the distributable archive (will be located in `./dist` directory)
+`python -m build --wheel`
+
 
 # TODO: how to use?
 ```
-
+```
 ## Donating
 
 [![Open Collective backers](https://img.shields.io/opencollective/all/CleverThis)](https://opencollective.com/cleverthis)

amqp/adapter/cleverthis_service_adapter.py

@@ -121,7 +121,7 @@ async def call_cleverthis_post_put_api(
 # =============  GET Helper function =============
 async def call_cleverthis_get_api(
         message: AMQMessage,
-        pattern: str,
+        pattern_or_data: str | dict,
         api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine],
         authenticated_user: Any
 ) -> AMQResponse:
@@ -134,14 +134,20 @@ async def call_cleverthis_get_api(
     :return:
     """
     try:
-        match = re.search(pattern, message.path)
-        if match:
-            crt: Coroutine = api_method(match.groups(), authenticated_user)
-            result: str = await crt
-            return AMQResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
+        args = None
+        if isinstance(pattern_or_data, dict):
+            args = pattern_or_data
         else:
-            _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern}"
-            return AMQResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
+            match = re.search(pattern_or_data, message.path)
+            if match:
+                args = match.groups()
+            else:
+                _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern_or_data}"
+                return AMQResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
+        crt: Coroutine = api_method(args, authenticated_user)
+        result: str = await crt
+        return AMQResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
+
     except HTTPException as http_error:
         tb = traceback.extract_tb(sys.exc_info()[2])[-1]
         logging.error(f" [E] Error in {tb.filename}, line {tb.lineno}, function '{tb.name}': {http_error}")

amqp/adapter/file_downloader.py

@@ -144,9 +144,10 @@ async def on_message(
         await asyncio.gather(*tasks) # Wait for all downloads to complete
 
         # After all files are downloaded, acknowledge the original message
-        logging.info(f"All files downloaded. Acknowledging the main message d-tag: {message.delivery_tag}")
-        await message.ack()
-        logging.info(f"Message d-tag: {message.delivery_tag} acknowledged.")
+        logging.info(f"All files downloaded. d-tag: {message.delivery_tag}")
+        #logging.info(f"All files downloaded. Acknowledging the main message d-tag: {message.delivery_tag}")
+        #await message.ack()
+        #logging.info(f"Message d-tag: {message.delivery_tag} acknowledged.")
     except Exception as e:
         logging.error(f"[E] [{message.delivery_tag}] Error processing message: {e}")
         tb = traceback.extract_tb(sys.exc_info()[2])[-1]

amqp/adapter/pydantic_serializer.py

@@ -0,0 +1,184 @@
+import uuid
+import json
+from datetime import datetime
+from typing import Dict, Any
+from pydantic import BaseModel
+from urllib.parse import urlparse, parse_qs
+
+
+def serialize_object(obj: BaseModel) -> str:
+    """
+    Serializes an R2RSerializable object to a string in the format
+    "<ClassName>:{key1:value1, key2:value2,...}".  Handles UUIDs and datetimes,
+    and nested Pydantic models.
+
+    Args:
+        obj: The BaseModel object to serialize.
+
+    Returns:
+        A string representation of the object.
+    """
+    class_name = obj.__class__.__name__
+    obj_dict = {}
+    for key, value in obj.model_dump().items():
+        if isinstance(value, uuid.UUID):
+            obj_dict[key] = str(value)
+        elif isinstance(value, datetime):
+            obj_dict[key] = value.isoformat()
+        elif isinstance(value, dict):  # Ensure dictionaries are handled
+            obj_dict[key] = value
+        elif isinstance(value, BaseModel):
+            obj_dict[key] = serialize_object(value)  # Recursively serialize
+        elif isinstance(value, list):
+            serialized_list = []
+            for item in value:
+                if isinstance(item, BaseModel):
+                    serialized_list.append(serialize_object(item))
+                else:
+                    serialized_list.append(item)
+            obj_dict[key] = serialized_list
+        else:
+            obj_dict[key] = value
+
+    dict_string = ",".join(f"{k}:{v}" for k, v in obj_dict.items())
+    return f"{class_name}:{{{dict_string}}}"
+
+
+def deserialize_object(data: str) -> BaseModel:
+    """
+    Deserializes a string in the format "<ClassName>:{key1:value1, key2:value2,...}"
+    back into an BaseModel object.  Handles UUIDs, datetimes, and
+    nested Pydantic models.
+
+    Args:
+        data: The string to deserialize.
+
+    Returns:
+        An R2RSerializable object.
+    """
+    class_name, dict_str = data.split(":", 1)
+    dict_str = dict_str.strip('{}')
+    pairs = dict_str.split(",") if dict_str else []
+
+    obj_dict: Dict[str, Any] = {}
+    for pair in pairs:
+        if not pair or ":" not in pair:
+            continue
+        k, v = pair.split(":", 1)
+        v = v.strip()  # important
+        obj_dict[k] = _convert_value(v)
+
+    print(f"Deserialized object: {class_name}")
+    if '[' in class_name:
+        class_name = class_name.split('[')[0]
+    print(f"trying object: {class_name}")
+    cls = globals().get(class_name)
+    if cls is None:
+        raise ValueError(f"Unknown class: {class_name}")
+
+    return cls(**obj_dict)
+
+
+def _convert_value(v: str) -> Any:
+    """
+    Helper function to convert a string to its appropriate Python type.
+    """
+    if v == 'None':
+        return None
+    elif _is_uuid_string(v):
+        return uuid.UUID(v)
+    elif _is_datetime_string(v):
+        return datetime.fromisoformat(v)
+    elif _is_int_string(v):
+        return int(v)
+    elif v.startswith("{") and v.endswith("}") and ":" in v and "{" not in v[1:-1] and "}" not in v[
+                                                                                                  1:-1]:  # nested object
+        # check if it is a json dict
+        try:
+            json.loads(v)
+            return json.loads(v)
+        except json.JSONDecodeError:
+            return deserialize_object(v)  # recursive call
+    elif v.startswith("[") and v.endswith("]"):  # list
+        v = v[1:-1]  # remove brackets
+        items = v.split(",") if v else []
+        deserialized_list = []
+        for item in items:
+            item = item.strip()
+            deserialized_list.append(_convert_value(item))  # recursive call
+        return deserialized_list
+    else:
+        return v  # Keep as string
+
+
+def _is_uuid_string(s: str) -> bool:
+    """Helper to check if a string is a valid UUID."""
+    try:
+        uuid.UUID(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_datetime_string(s: str) -> bool:
+    """Helper to check if a string is a valid datetime in ISO format"""
+    try:
+        datetime.fromisoformat(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_int_string(s: str) -> bool:
+    """Helper to check if a string is a valid int."""
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False
+
+
+def serialize_to_json(obj: BaseModel) -> str:
+    """
+    Serializes an BaseModel object to a JSON string.  Handles UUIDs and datetimes
+    natively using the default JSON encoder with some help for non-standard types.
+
+    Args:
+        obj: The R2RSerializable object to serialize.
+
+    Returns:
+        A JSON string representation of the object.
+    """
+    def default_serializer(o):
+        if isinstance(o, uuid.UUID):
+            return str(o)
+        elif isinstance(o, datetime):
+            return o.isoformat()
+        elif isinstance(o, BaseModel):
+            return o.model_dump()
+        raise TypeError(f"Object of type {o.__class__.__name__} is not JSON serializable")
+
+    return json.dumps(obj.model_dump(), default=default_serializer)
+
+
+def parse_url_query(url):
+    """
+    Parses a HTTP GET URL and returns (context_path, query_params_dict)
+
+    Args:
+        url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+    Returns:
+        tuple: (context_path, query_params_dict)
+        - context_path: The path part of the URL (e.g., "/path")
+        - query_params_dict: Dictionary of query parameters where values are always lists
+                           (e.g., {"foo": ["1", "3"], "bar": ["2"]})
+    """
+    parsed = urlparse(url)
+    query_params = parse_qs(parsed.query)
+
+    # Convert values from lists to single values when there's only one value
+    # If you prefer to always keep values as lists, remove this comprehension
+    simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
+
+    return (parsed.path, simplified_params)

amqp/model/model.py

@@ -204,3 +204,4 @@ class CleverMicroServiceMessage:
 
     def is_valid(self) -> bool:
         return self.message_type is not None
+

amqp/router/router_consumer.py

@@ -35,10 +35,10 @@ class RouterConsumer(RouterProducer):
         logging.info(" [*] RouterConsumer.initConsumerRouter, channel open: %s", self.is_open(_channel))
         if self.is_open(_channel):
             try:
-                #  1. declare a Queue that gives the RoutingDAta requests
+                #  1. declare a Queue to receive the RoutingData requests
                 cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
                 queue: AbstractRobustQueue = await self.channel.declare_queue(name=cm_request_queue, durable=True, exclusive=False, auto_delete=False)
-                # 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
+                # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key)
                 await queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
                 # 1b. Service adapter mode - listens on Request queue and replies to Response queue
                 logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue)
@@ -67,15 +67,17 @@ class RouterConsumer(RouterProducer):
                      delivery_tag, self.service_message_factory.to_string(cm_message), route_mapping)
         # Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
         if cm_message.is_valid() and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
-            try:
-                service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                    ServiceMessageType.ROUTING_DATA, route_mapping, cm_message.reply_to
-                )
-                await self.publish_service_message(service_message, self.cm_response_exchange)
-            except Exception as err:
-                logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
-                              delivery_tag, CM_RESPONSE_EXCHANGE, err)
-
+            if cm_message.reply_to != self.amq_configuration.amq_adapter.service_name:
+                try:
+                    service_message: CleverMicroServiceMessage = self.service_message_factory.of(
+                        ServiceMessageType.ROUTING_DATA, route_mapping, cm_message.reply_to
+                    )
+                    await self.publish_service_message(service_message, self.cm_response_exchange)
+                except Exception as err:
+                    logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
+                                  delivery_tag, CM_RESPONSE_EXCHANGE, err)
+            else:
+                logging.info(" [*] ******[%s] (ROUTING_DATA_REQ) [%s] Ignore request originated by ourselves.")
         else:
             logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
                           delivery_tag, str(message.body))

amqp/service/amq_service.py

@@ -35,7 +35,7 @@ class AMQService:
         logging.info("***********************************************")
         logging.info("**********  AMQ   Service   *******************")
         logging.info("***********************************************")
-        loop = asyncio.get_event_loop()
+        self.loop = asyncio.get_event_loop()
 
         self.amq_configuration = amq_configuration
         self.message_factory = AMQMessageFactory.get_instance(self.amq_configuration.amq_adapter.generator_id)
@@ -46,7 +46,7 @@ class AMQService:
             self.amq_configuration.amq_adapter.service_name
         )
         # self.init(loop) will initialize RabbitMQ connection
-        loop.run_until_complete(self.init(loop, service_adapter))
+        self.loop.run_until_complete(self.init(self.loop, service_adapter))
         # loop.run_until_complete(consume(loop))
         # self.consumer_thread = asyncio.create_task(self.rabbit_mq_consumer.channel.start_consuming())
         # self.rabbit_mq_consumer.request_routes()
@@ -54,7 +54,7 @@ class AMQService:
         logging.info("***********************************************")
         logging.info("******  AMQ   Service  Init  Complete *********")
         logging.info("***********************************************")
-        loop.run_forever()
+        self.loop.run_forever()
 
     async def init(self, loop, service_adapter):
         exchange: AbstractRobustExchange = await self.rabbit_mq_consumer.init(loop)
@@ -78,7 +78,7 @@ class AMQService:
     async def reinitialize_if_disconnected(self):
         if not self.rabbit_mq_consumer.channel or self.rabbit_mq_consumer.channel.is_closed:
             self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
-            await self.rabbit_mq_consumer.init(loop=None)
+            await self.rabbit_mq_consumer.init(loop=self.loop)
             await self.register_routes()
             await self.rabbit_mq_consumer.request_routes()
 

amqp/service/multipart_parser.py

@@ -32,6 +32,8 @@ class CleverMultiPartParser:
                 try:
                     self.form_data = json.loads(message.body())
                     self.is_json = True
+                    if self.form_data['files'] and self.form_data['form']:
+                        self.form_data = self.process_form_data(self.form_data)
                 except Exception as jsonerror:
                     logging.error(f"Error parsing JSON: {jsonerror}")
                     self.is_valid = False
@@ -48,3 +50,33 @@ class CleverMultiPartParser:
         self.form_data[file.field_name.decode('utf-8')] = \
             UploadFile(file.file_object, size=file.size, filename=file.file_name)
         print(self.form_data)
+
+    def process_form_data(self, json_input):
+        """
+        Processes the input JSON to extract form data, with files data overriding where available.
+
+        Args:
+            json_input: JSON string or dict containing the form and files data
+
+        Returns:
+            dict: Processed form data with files data overriding non-empty values
+        """
+        # Load JSON if input is string
+        if isinstance(json_input, str):
+            data = json.loads(json_input)
+        else:
+            data = json_input
+
+        result = {}
+
+        # Process form data - take first element of each list
+        for key, value in data['form'].items():
+            if value:  # Only process if list is not empty
+                result[key] = value[0]
+
+        # Override with files data where available
+        for key, value in data['files'].items():
+            if value:  # Only override if files data is non-empty
+                result[key] = value[0]
+
+        return result

pyproject.toml

@@ -2,15 +2,18 @@
 requires = ["setuptools>=61.0", "wheel"]  # Specifies build tools to use
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools]
+packages = ["amqp","amqp.adapter","amqp.config","amqp.model","amqp.rabbitmq","amqp.router","amqp.service"]  # Explicitly list packages to include
+
 [project]
 name = "amq_adapter"
-version = "0.2.2"
+version = "0.2.10"
 description = "The Python implementation of the AMQ Adaptor for CleverMicro"
 readme = "README.md"
 license = { text = "Apache License 2.0" }
 dependencies = [
     "aiohttp~=3.11.11",
-    "pika~=1.3.2",
+    "aio_pika~=9.5.5",
     "fastapi==0.115.6",
     "opentelemetry-api",
     "opentelemetry-sdk",

requirements.txt

@@ -1,8 +1,9 @@
 aiohttp~=3.11.11
 fastapi==0.115.6
-pika~=1.3.2
+aio_pika~=9.5.5
 opentelemetry-api
 opentelemetry-sdk
 opentelemetry-exporter-otlp
 opentelemetry-exporter-prometheus
 opentelemetry-instrumentation-pika
+python_multipart

setup.py

@@ -1,30 +0,0 @@
-from setuptools import setup, find_packages
-
-setup(
-    name="amqp",
-    version="0.2.0",
-    author="Stanislav Hejny",
-    author_email="stanislav.hejny@cleverthis.com",
-    description="Adapter for RabbitMQ to integrate a CleverThis python-code Service with CleverMicro platform",
-    long_description=open("README.md").read(),
-    long_description_content_type="text/markdown",
-    url="https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python",
-    packages=find_packages(),
-    install_requires=[
-        # dependencies here
-        "aiohttp~=3.11.11",
-        "fastapi==0.115.6",
-        "pika~=1.3.2",
-        "opentelemetry-api",
-        "opentelemetry-sdk",
-        "opentelemetry-exporter-otlp",
-        "opentelemetry-exporter-prometheus",
-        "opentelemetry-instrumentation-pika"
-    ],
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],
-    python_requires=">=3.11",
-)