#43 - fix the issues reported by flake8

Reviewed-on: #42

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+ignore = E501, W503, E203
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.venv

.forgejo/workflows/coverage-check.yaml

@@ -0,0 +1,47 @@
+name: "Unit test coverage"
+on:
+  push:
+  pull_request:
+env:
+  MIN_COVERAGE_PERCENTAGE: 85
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  # gradle test for modules
+  pytest:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git curl libsqlite3-0
+      # check out code and set up python
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest with coverage
+      - run: pytest --cov=amqp --cov-report=xml
+      # process coverage report
+      - name: Collect coverage report
+        id: coverage
+        run: |
+          COVERAGE_PERCENTAGE=$(coverage report --format=total)
+          echo "Code coverage: ${COVERAGE_PERCENTAGE}%"
+          echo "Minimal coverage: ${MIN_COVERAGE_PERCENTAGE}%"
+          echo "percentage=${COVERAGE_PERCENTAGE}" >> $GITHUB_OUTPUT
+      - name: Post coverage to RP
+        if: github.event_name == 'pull_request'
+        run: |
+          curl --fail \
+            -X POST '${{github.api_url}}/repos/${{github.repository}}/issues/${{github.event.number}}/comments' \
+            -H "Content-Type: application/json" \
+            -H "Authorization: token ${{github.token}}" \
+            -d '{"body":"Coverage is ${{steps.coverage.outputs.percentage}}%"}'
+      - name: Check coverage rate
+        run: |
+          coverage report --fail-under=${MIN_COVERAGE_PERCENTAGE}

.forgejo/workflows/publish-python.yaml

@@ -0,0 +1,50 @@
+name: "CI for pypl publish"
+on:
+  push:
+    branches:
+      - master # publish release on master
+      - develop # publish snapshot on develop
+  workflow_dispatch:
+  # allow manual trigger
+env:
+  DEBIAN_FRONTEND: noninteractive
+  TZ: UTC
+jobs:
+  publish-lib:
+    runs-on: general
+    container:
+      image: ubuntu:24.04
+    steps:
+      # need to setup node and git
+      - run: |
+          apt-get update
+          apt-get install -y nodejs git libsqlite3-0
+      # check out code
+      - uses: actions/checkout@v4
+      - uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      # install dependencies
+      - run: pip install -e .[dev]
+      # run pytest
+      - run: pytest
+      # replace timestamp building dev version
+      - name: Add timestamp to version
+        if: github.head_ref != 'master'
+        run: |
+          sed -i -E "s/(version = \"[^\"]+)(-dev)\"/\1\2$(date -u +'%Y%m%d%H%M%S')\"/" pyproject.toml
+          cat pyproject.toml
+      - name: Build wheel
+        run: |
+          python -m build
+      # publish
+      - name: publish to pulp pypi
+        run: |
+          twine upload --repository-url https://pkg.cleverthis.io/pypi/clevermicro/amq-adapter-python/simple/ \
+                       --username $CI_REGISTRY_USER \
+                       --password $CI_REGISTRY_PASSWORD \
+                       --verbose \
+                        dist/*
+        env:
+          CI_REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          CI_REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}

.gitignore

@@ -52,4 +52,11 @@ hs_err_pid*
 .idea
 .venv
 .coverage
+amq_adapter.egg-info/
+build/
 
+unused-code/
+
+amq_adapter.egg-info/
+build/
+coverage.xml

.gitlab-ci.yml

@@ -1,65 +0,0 @@
-# This pipeline would run the test coverage report to satisfy Coverage-Check rule.
-stages:
-  - test
-  - build
-  - publish
-
-variables:
-    PROJECT_ID: 217
-
-# Define a cache for pip dependencies to speed up builds
-cache:
-  paths:
-    - .cache/pip/
-
-# Install dependencies and run tests with coverage
-test:
-  stage: test
-  image: python:3.11  # Use the desired Python version
-  before_script:
-    - python -V  # Print Python version for debugging
-    - pip install --upgrade pip
-    - pip install -r requirements.txt  # Install project dependencies
-    - pip install pytest coverage  # Install testing and coverage tools
-  script:
-    - coverage run -m pytest  # Run tests with coverage
-    - coverage report -m  # Generate coverage report
-    - coverage xml  # Generate coverage report in XML format for GitLab
-  artifacts:
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage.xml
-    paths:
-      - coverage.xml
-  coverage: '/^TOTAL.*\s+(\d+%)$/'  # Regex to extract coverage percentage from the report
-  #rules:
-  #  - if: $CI_COMMIT_BRANCH == "develop"  # Run this job only on the 'develop' branch
-
-# Build the package
-build:
-  stage: build
-  image: python:3.11
-  before_script:
-    - pip install --upgrade pip
-    - pip install setuptools wheel
-  script:
-    - python setup.py sdist bdist_wheel
-  artifacts:
-    paths:
-      - dist/
-
-# Publish the package to PyPI
-publish:
-  stage: publish
-  image: python:3.11
-  before_script:
-    - pip install --upgrade pip
-    - pip install twine
-  script:
-    # - twine upload --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD dist/*
-    - twine upload --repository-url https://git.cleverthis.com/api/v4/projects/$PROJECT_ID/packages/pypi --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD dist/*
-  #only:
-  #  - main  # Only publish from the main branch
-  #rules:
-  #  - if: $CI_COMMIT_BRANCH == "main"

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/PyCQA/isort
+  rev: 6.0.1
+  hooks:
+  - id: isort
+    args: ["--profile", "black"]
+- repo: https://github.com/psf/black
+  rev: 25.1.0
+  hooks:
+  - id: black
+    language_version: python3.11
+- repo: https://github.com/PyCQA/flake8
+  rev: 7.2.0
+  hooks:
+  - id: flake8

README.md

@@ -15,28 +15,101 @@ for routing the traffic over the RabbitMQ to intended service and automated serv
 ### Install dependencies
 
 ```bash
-pip install -r requirements.txt
+pip install -e .[dev]
 ```
 
+This will install dependencies for developing, but won't install this project as an library.
+
+To build the wheel files:
+
+```bash
+python -m build
+```
+
+The wheel file should be located under `dist` folder.
+You can use `pip install /path/to/wheel.whl` to install this library.
+
 ### Use as a library
 
-Use this URL to add this project as a dependency:
-
-```
-git+https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python.git@develop#egg=amq_adapter
-```
-
-> Hint: use `git config --global credential.helper store` to enable Git credential manager
-> so you don't have to re-enter the gitlab credential everytime.
-
-Then:
+The library has been published to pulp registry at `https://pkg.cleverthis.io/pypi/clevermicro/amq-adapter-python/`.
+Configure the dependency `amq_adapter` with the above registry using
+your build tools, then:
 
 ```python
 from amqp.service import amq_service
 
-# TODO: how to use?
+# start using the lib
 ```
 
+#### How to build distributable Wheel archive
+
+install project requirements and dev dependencies
+`pip install -e .[dev]`
+
+update version in:
+`pyproject.toml`
+
+> For dev versions, keep the `-dev` suffix. The pipeline will replace it with a unique timestamp,
+> act like maven SNAPSHOT versions.
+
+build the distributable archive (will be located in `./dist` directory)
+`python -m build --wheel`
+
+# Integrating with actual CleverThis Service
+
+The following steps are required to integrate with the CleverThis service. This example uses ClverSwarm as specific use case,
+other python-based services would follow identical pattern.
+
+## Prerequisites
+IMPORTANT: AMQ Adapter relies on the service to use FastAPI to implement its REST endpoints
+and that those endpoints are properly annotated with FastAPI annotations.
+AMQ Adapter will look for these annotations to recognize the service API entry points and
+will automatically use the annotation to map the actual function to the incoming request the very same way FastAPI does.
+
+
+1. **Install the amq-adapter library**, as indicated above
+2. **Ensure the needed dependencies were installed** together with the amq-adapter. This is the list of dependencies:
+```
+aiohttp~=3.11.11
+fastapi==0.115.6
+aio_pika~=9.5.5
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-exporter-otlp
+opentelemetry-exporter-prometheus
+opentelemetry-instrumentation-pika
+python_multipart
+```
+
+Go to amq-dapter-python repo source code, and copy the service specific directory, e.g. `<git-root>/cleverswarm`
+directory to your service's sources. For the CleverSwarm service, the file is `cleverswarm/clever_swarm.py`, 
+and contains the AMQ Adapter to CleverSwarm service custom facade, implemented as
+```python
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+```
+
+to instatiate and initialize the AMQ adapter, add following line to the init routine of the CleverThis service:
+```python
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+```
+and ensure that the `adapter` stays in the context of the service, so it is not garbage collected.
+
+At version 0.2.21, AMQ Adapter uses local configuration file, relative path to which is passed 
+to the AMQConfiguration constructor. This may change in the future, when the central configuration service
+will become available in CleverMicro, but for now please ensure correct relative path wrt the current working directory.
+
+For the config entries, please refer to the wiki page:  [AMQ Adapter Config](https://docs.cleverthis.com/en/architecture/microservices/amq-adapter-configuration)
+
+### Environment variables
+
+The AMQ Adapter uses the following environment variables to configure the service:
+
+PARALLEL_WORKERS - number of parallel workers to use for processing incoming requests. This is currently mandatory value,
+shall be greater than 0.
+
+AMQ_VERBOSE_LOGGING - if set to 1, the adapter will indicate in the log next to the message itself also module name, line and function name where the message was generated
+
+```
 ## Donating
 
 [![Open Collective backers](https://img.shields.io/opencollective/all/CleverThis)](https://opencollective.com/cleverthis)

amqp/adapter/amq_route_factory.py

@@ -1,19 +1,23 @@
-import logging
-
+from amqp.adapter.logging_utils import logging_error
 from amqp.model.model import AMQRoute
 
+# record separator
 RS = ":"
+# null route, in lieu of None value
 NULL_ROUTE = {
     "componentName": "",
     "exchange": None,
     "queue": None,
     "routingKey": "#",
     "timeout": 1,
-    "validUntil": 0
+    "validUntil": 0,
 }
 
 
 class AMQRouteFactory:
+    """
+    Define the methods to create and validate AMQRoute objects.
+    """
 
     @staticmethod
     def from_string(data):
@@ -23,38 +27,41 @@ class AMQRouteFactory:
         :return: AMQRoute object
         """
         try:
-            keys = data.split(RS)
-            if len(keys) >= 5:
-                i = 0
-                component_name = keys[i]
-                i += 1
-                exchange = keys[i]
-                i += 1
-                queue = keys[i] if i < len(keys) else ""
-                i += 1
-                routing_key = keys[i] if i < len(keys) else ""
-                i += 1
-                timeout = int(keys[i]) if i < len(keys) else 0
-                i += 1
-                valid_until = int(keys[i]) if i < len(keys) else 0
+            _keys = data.split(RS)
+            if len(_keys) >= 5:
+                _i = 0
+                _component_name = _keys[_i]
+                _i += 1
+                _exchange = _keys[_i]
+                _i += 1
+                _queue = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _routing_key = _keys[_i] if _i < len(_keys) else ""
+                _i += 1
+                _timeout = int(_keys[_i]) if _i < len(_keys) else 0
+                _i += 1
+                _valid_until = int(_keys[_i]) if _i < len(_keys) else 0
                 return AMQRoute(
-                    component_name=component_name,
-                    exchange=exchange,
-                    queue=queue,
-                    key=routing_key,
-                    timeout=timeout,
-                    valid_until=valid_until
+                    component_name=_component_name,
+                    exchange=_exchange,
+                    queue=_queue,
+                    key=_routing_key,
+                    timeout=_timeout,
+                    valid_until=_valid_until,
                 )
-            logging.error(
-                " [E] AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", data
+                "got: ['%s']",
+                data,
             )
         except Exception as err:
-            logging.error(
-                " [E] %s: AMQRoute incorrect format. AMQRoute expects input as: "
+            logging_error(
+                "%s: AMQRoute incorrect format. AMQRoute expects input as: "
                 "'Component-Name:AMQExchange-Name:Queue-Name:Routing-Key:Timeout:Valid-Until', "
-                "got: ['%s']", err, data
+                "got: ['%s']",
+                err,
+                data,
             )
         return NULL_ROUTE
 

amqp/adapter/backpressure_handler.py

@@ -0,0 +1,324 @@
+import asyncio
+import json
+import time
+from asyncio import AbstractEventLoop
+from threading import Thread
+
+import psutil
+from aio_pika import Message
+from aio_pika.abc import AbstractRobustChannel
+
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import ScalingRequestAlert
+from amqp.router.utils import await_future, await_result
+
+
+class ScaleRequestV1:
+
+    def __init__(
+        self,
+        serviceId: str,
+        taskId: str,
+        maxAvailability: int,
+        currentAvailability: int,
+        requestType: ScalingRequestAlert,
+    ):
+        self.version = 1  # Version of the request, currently 1
+        self.serviceId = serviceId
+        self.taskId = taskId
+        self.maxAvailability = maxAvailability
+        self.currentAvailability = currentAvailability
+        self.requestType = requestType
+        self.thread = None
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.serviceId,
+                "taskId": self.taskId,
+                "maxAvailability": self.maxAvailability,
+                "currentAvailability": self.currentAvailability,
+                "requestType": self.requestType.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+
+class RunningAverage:
+    def __init__(self, num_items):
+        self.buffer = [0] * num_items  # Fixed-size array
+        self.pointer = 0  # Current position in array
+        self.num_items = num_items  # Maximum number of items to store
+        self.is_filled = False  # Track if buffer is fully filled
+
+    def add(self, value):
+        # Store the value at current pointer position
+        self.buffer[self.pointer] = value
+
+        # Move pointer to next position with wrap-around
+        self.pointer += 1
+        if self.pointer >= self.num_items:
+            self.pointer = 0
+            self.is_filled = True
+
+    def get_average(self):
+        # Determine how many items we should average
+        count = self.num_items if self.is_filled else self.pointer
+
+        if count == 0:
+            return 0.0  # Avoid division by zero
+
+        return sum(self.buffer) / count
+
+    def get_current_values(self):
+        """Returns the values in chronological order (oldest first)"""
+        if not self.is_filled:
+            return self.buffer[: self.pointer]
+
+        return self.buffer[self.pointer :] + self.buffer[: self.pointer]
+
+
+class BackpressureHandler:
+    # Track the number of messages currently being processed
+    current_parallel_executions = 0
+    # helps detect IDLE condition
+    last_data_message_time = 0
+    # helps prevent flooding the system with backpressure events
+    last_backpressure_event_time = 0
+    last_backpressure_event = ScalingRequestAlert.UPDATE
+
+    def __init__(
+        self,
+        channel: AbstractRobustChannel,
+        loop: AbstractEventLoop,
+        config: AMQConfiguration,
+    ):
+        self.lock = None
+        self.channel = channel
+        self.loop = loop
+        self.config = config
+        self.exchange = None
+        self.swarm_service_id = self.config.amq_adapter.swarm_service_id
+        self.swarm_task_id = self.config.amq_adapter.swarm_task_id
+        self.parallel_workers = self.config.backpressure.threshold_threads
+        self.average_cpu_usage = None
+
+    def increase_parallel_executions(self):
+        """Increase the number of parallel executions"""
+        logging_info(
+            "Backpressure: Increase parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        self.current_parallel_executions += 1
+
+    def decrease_parallel_executions(self):
+        """Decrease the number of parallel executions"""
+        logging_info(
+            "Backpressure: Decrease parallel executions, current=%s",
+            self.current_parallel_executions,
+        )
+        if self.current_parallel_executions > 0:
+            self.current_parallel_executions -= 1
+
+    def update_parallel_executions(self, count: int):
+        """Update the number of parallel executions"""
+        self.current_parallel_executions = count
+
+    def update_last_data_message_time(self):
+        logging_info("Backpressure: Update last data message time")
+        """Update the last data message time"""
+        self.last_data_message_time = time.time()
+
+    def start_backpressure_monitor(self) -> Thread:
+        # Start the Backpressure monitor loop
+        self.thread = Thread(target=self.backpressure_monitor_loop)
+        self.thread.start()
+        return self.thread
+
+    async def check_overload_condition(self):
+        """Check if the current parallel executions exceed the limit"""
+        self.update_last_data_message_time()
+        logging_info(
+            "Backpressure: Check overload condition, current=%s, max=%s",
+            self.current_parallel_executions,
+            self.parallel_workers,
+        )
+        if self.current_parallel_executions >= self.parallel_workers - 1:
+            # Check if the last backpressure event was not an overload
+            if self.last_backpressure_event != ScalingRequestAlert.OVERLOAD:
+                # Trigger the overload event
+                await self.handle_backpressure_overload_event()
+                self.last_backpressure_event_time = time.time()
+                self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+
+    def backpressure_monitor_loop(self):
+        _time_window_millis = self.config.backpressure.time_window
+        _idle_duration_millis = self.config.backpressure.idle_duration
+        _overload_duration_millis = self.config.backpressure.idle_duration
+        _loop = asyncio.new_event_loop()
+        _monitor_interval = 0.2  # Monitor every 200ms
+        CPU_IDLE, CPU_ACTIVE, CPU_OVERLOAD = 0, 1, 2
+        IDLE_THRESHOLD, OVERLOAD_THRESHOLD = 15, 90
+        self.average_cpu_usage = RunningAverage(
+            int(
+                max(_overload_duration_millis, _idle_duration_millis)
+                // _monitor_interval
+            )
+        )
+
+        async def _backpressure_monitor():
+            """Monitor the backpressure conditions and trigger events accordingly"""
+            _cpu_usage_state = CPU_ACTIVE
+            _cpu_usage_changed = time.time()
+            while True:
+                _current_cpu_usage = psutil.cpu_percent(interval=None)
+                self.average_cpu_usage.add(_current_cpu_usage)
+                _average_cpu_usage = self.average_cpu_usage.get_average()
+                _now = time.time()
+                # logging_info(
+                #    "Backpressure: current CPU[pct]=%s, avg=%s, state=%s, last dataMsg=%s, last event=%s, last eventTime=%s",
+                #    _current_cpu_usage,
+                #    _average_cpu_usage,
+                #    _cpu_usage_state,
+                #    self.last_data_message_time,
+                #    self.last_backpressure_event,
+                #    self.last_backpressure_event_time,
+                # )
+                if _average_cpu_usage < IDLE_THRESHOLD:
+                    if _cpu_usage_state != CPU_IDLE:
+                        _cpu_usage_changed = _now
+                    _cpu_usage_state = CPU_IDLE
+                elif _average_cpu_usage > OVERLOAD_THRESHOLD:
+                    if _cpu_usage_state != CPU_OVERLOAD:
+                        _cpu_usage_changed = _now
+                    _cpu_usage_state = CPU_OVERLOAD
+                else:
+                    if _cpu_usage_state != CPU_ACTIVE:
+                        _cpu_usage_changed = _now
+                    _cpu_usage_state = CPU_ACTIVE
+
+                # Check if the current time exceeds the last backpressure event time
+                if (
+                    _cpu_usage_state == CPU_OVERLOAD
+                    and _now - _cpu_usage_changed > _overload_duration_millis
+                    and (
+                        _now - self.last_backpressure_event_time > _time_window_millis
+                        or self.last_backpressure_event != ScalingRequestAlert.OVERLOAD
+                    )
+                ):
+                    # Trigger the overload event
+                    await self.handle_backpressure_overload_event()
+                    self.last_backpressure_event_time = _now
+                    self.last_backpressure_event = ScalingRequestAlert.OVERLOAD
+                elif (
+                    _cpu_usage_state == CPU_IDLE
+                    and _now - _cpu_usage_changed > _idle_duration_millis
+                    and _now - self.last_data_message_time > _idle_duration_millis
+                    and (
+                        _now - self.last_backpressure_event_time > _time_window_millis
+                        or self.last_backpressure_event != ScalingRequestAlert.IDLE
+                    )
+                ):
+                    # Trigger the idle event
+                    await self._handle_backpressure_idle_event()
+                    self.last_backpressure_event = ScalingRequestAlert.IDLE
+                    self.last_backpressure_event_time = _now
+                else:
+                    # Trigger update event
+                    if _now - self.last_backpressure_event_time > _time_window_millis:
+                        # Trigger the update event
+                        _scaling_request: ScaleRequestV1 = ScaleRequestV1(
+                            self.swarm_service_id,
+                            self.swarm_task_id,
+                            100,
+                            _average_cpu_usage,
+                            ScalingRequestAlert.UPDATE,
+                        )
+                        # Address the message to any adapter capable of supporting BACKPRESSURE request
+                        await self.publish_backpressure_request(_scaling_request)
+                        self.last_backpressure_event_time = _now
+                        self.last_backpressure_event = ScalingRequestAlert.UPDATE
+                await asyncio.sleep(_monitor_interval)
+
+        _loop.run_until_complete(_backpressure_monitor())
+
+    async def handle_backpressure_overload_event(self):
+        logging_warning("Backpressure: Capacity close to depleted!")
+        # Send an Overload event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.OVERLOAD,
+        )
+        # Address the message to any management service instance capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the OVERLOAD is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def _handle_backpressure_idle_event(self):
+        logging_warning("Backpressure: Service is idle.")
+        # Send an Idle event to the Management service
+        scaling_request: ScaleRequestV1 = ScaleRequestV1(
+            self.swarm_service_id,
+            self.swarm_task_id,
+            self.parallel_workers,
+            self.parallel_workers - self.current_parallel_executions,
+            ScalingRequestAlert.IDLE,
+        )
+        # Address the message to any adapter capable of supporting BACKPRESSURE request
+        await self.publish_backpressure_request(scaling_request)
+        # update / reset time-window so that the IDLE is not sent too often
+        self.last_data_message_time = time.time()
+
+    async def publish_backpressure_request(self, scaling_request: ScaleRequestV1):
+        # Publish the backpressure request to the management service
+        logging_info(
+            f"Publishing backpressure for {scaling_request.serviceId} with request type {scaling_request.requestType}"
+        )
+        if not self.exchange:
+
+            async def _wrap_rabbit_mq_api_init(channel):
+                _exchange = await channel.get_exchange(
+                    name="cleverthis.clevermicro.management"
+                )
+                return _exchange
+
+            self.exchange = await await_result(
+                asyncio.run_coroutine_threadsafe(
+                    _wrap_rabbit_mq_api_init(self.channel), self.loop
+                )
+            )
+
+        if self.exchange:
+
+            async def _wrap_rabbit_mq_api():
+                if not self.channel.is_closed:
+                    binary_content: bytes = scaling_request.to_json().encode("utf-8")
+                    pika_message: Message = Message(
+                        body=binary_content,
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        content_type="application/octet-stream",
+                        headers=None,
+                        priority=0,
+                        correlation_id=None,
+                    )
+                    await self.exchange.publish(
+                        message=pika_message, routing_key="backpressure-scaling-v1"
+                    )
+                    logging_info(
+                        "Service Message Published to %s, msg: %s",
+                        self.exchange.name,
+                        str(binary_content),
+                    )
+                    return True
+                return False
+
+            await await_future(
+                asyncio.run_coroutine_threadsafe(_wrap_rabbit_mq_api(), self.loop)
+            )

amqp/adapter/cleverthis_service_adapter.py

@@ -1,140 +1,122 @@
+"""
+The layer that interfaces with the CleverThis service that this Adapter is integrating with.
+Methods here will invoke the actual CleverThis Service code, which are passed as Callable object.
+"""
+
 import asyncio
+import inspect
 import json
-import logging
-import os
-import re
-import threading
+from asyncio import AbstractEventLoop
+from dataclasses import dataclass
+from typing import Any, Callable, Coroutine, Dict, List
 
-from aiohttp import ClientSession, ClientResponse
+from aio_pika.abc import (
+    AbstractExchange,
+    AbstractRobustChannel,
+    AbstractRobustConnection,
+)
+from aiohttp import ClientResponse, ClientSession
 from fastapi import HTTPException
-from typing import Dict, List, Tuple, Callable, Coroutine, AnyStr, Any
-from urllib.parse import parse_qs
-from xml.etree import ElementTree
+from pydantic import BaseModel
+from starlette.responses import FileResponse
 
-from amqp.adapter.amq_response_factory import AMQResponseFactory
-from amqp.service.multipart_parser import CleverMultiPartParser
+from amqp.adapter import serializer
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage, AMQResponse
-
-# Track the number of messages currently being processed
-current_parallel_executions = 0
-lock = threading.Lock()
-parallel_workers = os.getenv("PARALLEL_WORKERS", default=10)
+from amqp.model.model import AMQErrorMessage, DataMessage, DataResponse
+from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
+from amqp.router.utils import await_result
 
 
-def parse_request_body(message: AMQMessage):
+@dataclass
+class AMQMessage(DataMessage):
     """
-    Parses the HTTP POST request body based on the MIME type.
-
-    Args:
-        message (AMQMessage): Contains raw request and MIME headers.
-
-    Returns:
-        dict: Parsed data as a dictionary.
+    Expand the core DataMessage type to include the channel, which is required to create a dedicated queue
+    in case the response should initiate file download.
     """
-    content_type = message.headers.get('Content-Type', '')
-    if not content_type:
-        raise ValueError("Content-Type header is required")
 
-    # Parse JSON
-    if content_type == "application/json":
-        try:
-            return json.loads(message.body())
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON: {e}")
-
-    # Parse URL-encoded form data
-    elif content_type == "application/x-www-form-urlencoded":
-        try:
-            return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(message.body()).items()}
-        except Exception as e:
-            raise ValueError(f"Invalid URL-encoded form data: {e}")
-
-    # Parse multipart/form-data
-    elif content_type.startswith("multipart/form-data"):
-        try:
-            parser: CleverMultiPartParser = CleverMultiPartParser(message)
-            return parser.form_data
-        except Exception as e:
-            raise ValueError(f"Invalid multipart/form-data: {e}")
-
-    # Parse plain text
-    elif content_type == "text/plain":
-        return {"text": message.body()}
-
-    # Parse XML
-    elif content_type == "application/xml":
-        try:
-            root = ElementTree.fromstring(message.body())
-            return {elem.tag: elem.text for elem in root}
-        except ElementTree.ParseError as e:
-            raise ValueError(f"Invalid XML: {e}")
-
-    else:
-        raise ValueError(f"Unsupported Content-Type: {content_type}")
-
-
-# =============  POST Helper function =============
-async def call_cleverthis_post_put_api(
-        message: AMQMessage,
-        args: Any,
-        api_method: Callable[[Any, Any], Coroutine],
-        success_code: int,
-        authenticated_user: Any
-) -> AMQResponse:
-    """
-    Invokes the provided API method with the given arguments and authenticated user,
-    and returns the response as AMQResponse.
-    :param message: AMQMessage containing the details of the call and the payload
-    :param args: map of the arguments to pass to the API method
-    :param api_method: Callable - the method to invoke. Last parameter is the authenticated user
-    :param success_code: HTTP code to be returned when operation suceeds
-    :param authenticated_user: Authenticated user. The actual class depends on the service
-    :return: AMQResponse class
-    """
-    try:
-        coroutine_call: Coroutine = api_method(args, authenticated_user)
-        result: str = await coroutine_call
-        return AMQResponseFactory.of(
-            message.id, success_code, message.headers.get("Content-Type", "application/json"),
-            result.encode('utf-8'), message.trace_info
+    def __init__(self, data_message: DataMessage, connection: AbstractRobustConnection):
+        # Copy all fields from DataMessage
+        self.magic = data_message.magic
+        self.id = data_message.id
+        self.method = data_message.method
+        self.domain = data_message.domain
+        self.path = data_message.path
+        self.headers = data_message.headers
+        self.trace_info = data_message.trace_info
+        self.base64_body = data_message.base64_body
+        # Add the new field
+        self.connection = connection
+        self.extra_data = (
+            data_message.extra_data if hasattr(data_message, "extra_data") else {}
         )
-    except HTTPException as http_error:
-        return AMQResponseFactory.of(message.id, http_error.status_code, "text/plain",
-                                     str(http_error.detail).encode('utf-8'), message.trace_info)
-    except Exception as error:
-        return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
 
 
-# =============  GET Helper function =============
-async def call_cleverthis_get_api(
-        message: AMQMessage,
-        pattern: str,
-        api_method: Callable[[Tuple[AnyStr | Any, ...], Any], Coroutine],
-        authenticated_user: Any
-) -> AMQResponse:
+async def _convert_file_response(
+    obj: FileResponse,
+    connection: AbstractRobustConnection,
+    file_handler: FileHandler,
+    loop: AbstractEventLoop | None,
+) -> str:
     """
-    invokes provided Callable that is expected to be a GET API method, and returns the response as AMQResponse
-    :param message: AMQMessage containing the details of the call
-    :param pattern: If there are path variables, use this pattern to extract them
-    :param api_method: Callable - the method to invoke. Last parameter is the authenticated user
-    :param authenticated_user: Authenticated user. The actual class depends on the service
-    :return:
+    Converts a FileResponse object to a JSON string containing the filename and stream name.
+    The file content itself is transmitted through the named stream (which is implemented as temporary dedicated queue.
     """
-    try:
-        match = re.search(pattern, message.path)
-        if match:
-            crt: Coroutine = api_method(match.groups(), authenticated_user)
-            result: str = await crt
-            return AMQResponseFactory.of(message.id, 200, "text/plain", result.encode('utf-8'), message.trace_info)
-        else:
-            _msg = f"Unexpected URL: {message.path}, expected pattern: {pattern}"
-            return AMQResponseFactory.of(message.id, 404, "text/plain", _msg.encode('utf-8'), message.trace_info)
-    except HTTPException as http_error:
-        return AMQResponseFactory.of(message.id, http_error.status_code, "text/plain",
-                                     str(http_error.detail).encode('utf-8'), message.trace_info)
-    except Exception as error:
-        return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
+
+    async def _wrap_amq_api_calls(
+        connection: AbstractRobustConnection,
+    ) -> tuple[AbstractRobustChannel, AbstractExchange, str]:
+        # Wrap calls to RabbitMQ API to make it Awaitable, so it can be run thread-safe with correct event loop
+        # This is needed because the RabbitMQ API is not thread-safe and needs to be called in the context of
+        # the event loop the first RabbitMQ connection was created with.
+        _channel = await connection.channel()
+        _exchange_name: str = AMQ_REPLY_TO_EXCHANGE
+        _exchange = await _channel.get_exchange(name=_exchange_name, ensure=True)
+        _queue = await _channel.declare_queue(
+            exclusive=False
+        )  # Declare a unique, non-exclusive queue
+        _queue_name = _queue.name
+        await _queue.bind(
+            exchange=_exchange_name,
+            routing_key=_queue_name,  # Use queue name as routing key
+        )
+        logging_debug(f"Publishing to exchange: {_exchange_name}, q: {_queue_name}")
+        return _channel, _exchange, _queue_name
+
+    # All calls to RabbitMQ API have to be called threadsafe, because those appear to be attached to parent
+    # event loop, and need to execute at that event loop, not the current one
+    # allow coroutine to execute. calling _future.result() will block the entire thread and the coroutine will never execute
+    _channel, _exchange, _routing_key = await await_result(
+        asyncio.run_coroutine_threadsafe(_wrap_amq_api_calls(connection), loop)
+    )
+    # publish_file also needs RabbitMQ API, ensure the loop is passed as argument
+    (_stream_name, _file_size) = await file_handler.publish_file(
+        _exchange, obj.path, _routing_key, loop
+    )
+    # as above, but luckily this time we don't need to wait for the result
+    asyncio.run_coroutine_threadsafe(_channel.close(), loop)
+    return json.dumps(
+        {
+            "files": [
+                {
+                    "filename": obj.filename,
+                    "mediaType": "application/octet-stream",
+                    "streamName": _stream_name,
+                    "fileSize": _file_size,
+                }
+            ]
+        }
+    )
+
+
+def is_likely_json(text: str) -> bool:
+    """Quick test to see if should convert the response string to JSON format."""
+    _text = text.strip()
+    return (_text.startswith("{") and _text.endswith("}")) or (
+        _text.startswith("[") and _text.endswith("]")
+    )
 
 
 # =============  CleverThisServiceAdapter =============
@@ -142,159 +124,327 @@ class CleverThisServiceAdapter:
     """
     A base class for CleverThis service adapter that handles the incoming AMQP messages and routes them to the
     appropriate CleverThisService API endpoint.
+    IMPORTANT: This class is not intended to be used directly. It should be subclassed for each CleverThis service
+    and override the methods to handle the specific API calls.
     """
-    def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
+
+    def __init__(
+        self, amq_configuration: AMQConfiguration, session: ClientSession = None
+    ):
+        """
+        Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session.
+        The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via
+        HTTP REST API.
+        """
         self.amq_configuration = amq_configuration
         self.session = session
         self.service_host = self.amq_configuration.dispatch.service_host
         self.service_port = self.amq_configuration.dispatch.service_port
+        self.loop: AbstractEventLoop | None = None
+        self.require_authenticated_user = (
+            self.amq_configuration.amq_adapter.require_authenticated_user
+        )
 
-    async def get_current_user(self, token: str):
+    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
+        """
+        Pull the value of the Content-Type header from the message headers.
+        :param message_headers: The headers of the message.
+        return: The content type of the message.
+        """
+        for header, values in message_headers.items():
+            if header.lower() == "content-type":
+                return values[0].split(";")[0]
+        return "application/json"
+
+    async def get_current_user(self, token: str) -> object:
         """
         To Be Overriden in subclass for given CleverThis Service, to return the current user based on the token,
         in format appropriate for the service.
+        :param token: The token to be used for authentication.
         :return: A user object or None if the user is not authenticated.
         """
         return None
 
-    def sync_on_message(self, message: AMQMessage) -> AMQResponse:
-        """
-        Synchronous version of on_message method. This is a wrapper around the async on_message method.
-        :param message: AMQMessage
-        :return: AMQResponse
-        """
-        return asyncio.run(self.on_message(message))
-
-    async def on_message(self, message) -> AMQResponse:
+    async def on_message(self, message: AMQMessage) -> DataResponse:
         """
         Called when a message is received from the AMQP.
         :param message:
         :return: AMQP response class with operation status code and optional error details.
         """
-        DEBUG_NO_AUTH = 1
-        global current_parallel_executions
-
-        # Increment the counter for parallel executions
-        with lock:
-            current_parallel_executions += 1
-            if current_parallel_executions > parallel_workers - 2:
-                logging.warn("Warning: CApacity close to depleted!")
-
-        _auth: str = message.headers.get('Authorization', '')
         _auth_user: Any | None = None
-        if 'Bearer' in _auth:
-            try:
-                token = _auth.split(' ')[1]
-                _auth_user = await self.get_current_user(token)
-            except Exception as error:
-                logging.error(f"Error getting user: {error}")
-        amq_response: AMQResponse | None = None
-        if _auth_user or DEBUG_NO_AUTH == 1:
+        _amq_response: Any | None = None
+
+        logging_debug(
+            f"on_message: require_authenticated_user={self.require_authenticated_user}"
+        )
+        if self.require_authenticated_user:
+            _auth: str = message.headers.get("Authorization", "")
+            logging_debug(f"Auth: {_auth}")
+            if isinstance(_auth, List):
+                _auth = _auth[0]
+            if "Bearer" in _auth:
+                try:
+                    token = _auth.split(" ")[1]
+                    logging_info(f"Token: {token}")
+                    _auth_user = await self.get_current_user(token)
+                except Exception as error:
+                    logging_error(f"on_message: Error getting user: {error}")
+
+        if (
+            _auth_user
+            or not self.require_authenticated_user
+            or message.path.endswith("/login")
+        ):
             method = message.method.lower()
-            if method == 'get':
-                amq_response = await self.get(message, _auth_user)
-            elif method == 'put':
-                amq_response = await self.put(message, _auth_user)
-            elif method == 'post':
-                amq_response = await self.post(message, _auth_user)
-            elif method == 'head':
-                amq_response = await self.head(message, _auth_user)
-            elif method == 'delete':
-                amq_response = await self.delete(message, _auth_user)
+            if method == "get":
+                _amq_response = await self.get(message, _auth_user)
+            elif method == "put":
+                _amq_response = await self.put(message, _auth_user)
+            elif method == "post":
+                _amq_response = await self.post(message, _auth_user)
+            elif method == "head":
+                _amq_response = await self.head(message, _auth_user)
+            elif method == "delete":
+                _amq_response = await self.delete(message, _auth_user)
             else:
                 raise ValueError(f"Unexpected HTTP method: {message.method}")
         else:
-            amq_response = AMQResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
-        with lock:
-            current_parallel_executions -= 1
-        return amq_response
+            _amq_response = DataResponseFactory.of_error_message(
+                message.id,
+                AMQErrorMessage(401, "Unauthorized", "Unauthorized"),
+                message.trace_info,
+            )
+        logging_info(
+            f"ALL DONE in on_message id:{message.id}, resp code:{_amq_response.response_code}"
+        )
+        return _amq_response
 
     # ==================================================================================================
     # =====================  C l e v e r T h i s    A P I    m e t h o d s  ============================
     # ==================================================================================================
 
-    async def get(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
+    async def get(self, message: DataMessage, auth_user: Any) -> DataResponse:
         """
         Handles the GET requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
         It will attempt to call the REST endpoint directly using HTTP protocol.
         :param message: message from the AMQP that contains the GET request
         :param auth_user: user authenticated by the API Gateway, as UserSchem
-        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        :return: DataResponse to be sent back to the API Gateway via AMQP
         """
 
         # unmatched path - try to invoke the service directly on this path
         url = f"http://{self.service_host}:{self.service_port}{message.path}"
         headers = {**message.headers, **message.trace_info}
-        logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
+        logging_info(
+            f"SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}"
+        )
         if self.session:
             _http_resp: ClientResponse = await self.session.get(url, headers=headers)
-            return AMQResponseFactory.of(message.id, _http_resp.status, _http_resp.content_type,
-                                         await _http_resp.read(),
-                                         message.trace_info)
-        return AMQResponseFactory.of(message.id, 404, "text/plain",
-                                     str("Destination location does not exists").encode('utf-8'), message.trace_info)
+            return DataResponseFactory.of(
+                message.id,
+                _http_resp.status,
+                _http_resp.content_type,
+                await _http_resp.read(),
+                message.trace_info,
+            )
+        return DataResponseFactory.of_error_message(
+            message.id,
+            AMQErrorMessage(404, "Not Found", "Destination location does not exists"),
+            message.trace_info,
+        )
 
-    async def put(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
+    async def put(self, message: DataMessage, auth_user: Any) -> DataResponse:
         """
         Handles the PUT requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
         It will attempt to call the REST endpoint directly using HTTP protocol.
         :param message: message from the AMQP that contains the PUT request
         :param auth_user: user authenticated by the API Gateway, as UserSchem
-        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        :return: DataResponse to be sent back to the API Gateway via AMQP
         """
-        return await self.handle_possible_form(
+        return await self._authorize_forward_request(
+            message,
             self.session.put(
                 f"http://{self.service_host}:{self.service_port}{message.path}",
                 data=message.body,
                 headers={**message.headers, **message.trace_info},
             ),
-            auth_user
+            auth_user,
         )
 
-    async def post(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
+    async def post(self, message: DataMessage, auth_user: Any) -> DataResponse:
         """
         Handles the POST requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
         It will attempt to call the REST endpoint directly using HTTP protocol.
         :param message: message from the AMQP that contains the POST request, including body and headers
         :param auth_user: user authenticated by the API Gateway, as UserSchem
-        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        :return: DataResponse to be sent back to the API Gateway via AMQP
         """
-        return await self.handle_possible_form(
+        return await self._authorize_forward_request(
             message,
             self.session.post(
                 f"http://{self.service_host}:{self.service_port}{message.path}",
                 data=message.body,
                 headers={**message.headers, **message.trace_info},
             ),
-            auth_user
+            auth_user,
         )
 
-    async def handle_possible_form(self, message: AMQMessage, request_coroutine, auth_user: Any) -> AMQResponse:
-        args = parse_request_body(message)
+    async def _authorize_forward_request(
+        self, message: DataMessage, request_coroutine, auth_user: Any
+    ) -> DataResponse:
+        """
+        This method is used to authorize the request and forward it to the CleverThis Service API.
+        However, it is only used for loose / 3rd party mode of coupling with the Service
+        """
+        # AMQDataParser.parse_request_body(message)
         return await request_coroutine
 
-    async def head(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
+    async def head(self, message: DataMessage, auth_user: Any) -> DataResponse:
         """
         Handles the HEAD requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
         It will attempt to call the REST endpoint directly using HTTP protocol.
         :param message: message from the AMQP that contains the POST request, including body and headers
         :param auth_user: user authenticated by the API Gateway, as UserSchem
-        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        :return: DataResponse to be sent back to the API Gateway via AMQP
         """
         raise NotImplementedError
 
-    async def delete(self, message: AMQMessage, auth_user: Any) -> AMQResponse:
+    async def delete(self, message: DataMessage, auth_user: Any) -> DataResponse:
         """
         Handles the DELETE requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
         It will attempt to call the REST endpoint directly using HTTP protocol.
         :param message: message from the AMQP that contains the POST request, including body and headers
         :param auth_user: user authenticated by the API Gateway, as UserSchem
-        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        :return: DataResponse to be sent back to the API Gateway via AMQP
         """
         raise NotImplementedError
 
-    def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
-        for header, values in message_headers.items():
-            if header.lower() == 'content-type':
-                return values[0].split(';')[0]
-        return 'application/json'
+    @staticmethod
+    async def call_cleverthis_api(
+        message: AMQMessage,
+        args: Any,
+        api_method: Callable[[Any, Any], Coroutine],
+        success_code: int,
+        loop: AbstractEventLoop | None = None,
+    ) -> DataResponse:
+        """
+        Handles POST and PUT requests, which typically contain possibly complex payload data, passed as a dictionary.
+        Invokes the provided API method with the given arguments and authenticated user,
+        and returns the response as DataResponse.
+        :param message: DataMessage containing the details of the call and the payload
+        :param args: map of the arguments to pass to the API method
+        :param api_method: Callable - the method to invoke. Last parameter is the authenticated user
+        :param success_code: HTTP code to be returned when operation suceeds
+        :return: DataResponse class
+        """
+        try:
+            _verified_args = {}
+            _sig = inspect.signature(api_method)
+            _required_args = list(_sig.parameters.values())
+            # _required_args = getattr(api_method, '__annotations__', {})
+            for arg in _required_args:
+                if arg.name in args:
+                    # convert to another type if arg is not a str
+                    if arg.annotation == int:
+                        _verified_args[arg.name] = int(args[arg.name])
+                    elif arg.annotation == float:
+                        _verified_args[arg.name] = float(args[arg.name])
+                    elif arg.annotation == bool:
+                        _verified_args[arg.name] = serializer.str_to_bool(
+                            args[arg.name]
+                        )
+                    # TODO: more?
+                    else:
+                        _verified_args[arg.name] = args[arg.name]
+                else:
+                    logging_info(
+                        f"{api_method.__name__}: Missing required argument: {arg.name}"
+                    )
+            logging_info(
+                f"INVOKE ENDPOINT: {api_method.__name__}, ARGS: {', '.join(f'{k}={v}' for k, v in _verified_args.items())}"
+            )
+            _result: Any = await api_method(**_verified_args)
+            logging_info(
+                f"ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}"
+            )
+
+            if isinstance(_result, FileResponse):
+                _json_result, _content_type = (
+                    await _convert_file_response(
+                        _result,
+                        message.connection,
+                        file_handler=StreamingFileHandler(),
+                        loop=loop,
+                    ),
+                    "application/octet-stream",
+                )
+            elif isinstance(_result, BaseModel):
+                _json_result = _result.json()
+                _content_type = "application/json"
+            else:
+                _json_result = _result
+                _content_type = "application/json"
+
+            _binary_result = (
+                _json_result.encode("utf-8")
+                if hasattr(_json_result, "encode")
+                else (
+                    _json_result.body
+                    if hasattr(_json_result, "body")
+                    else (
+                        _json_result.read()
+                        if hasattr(_json_result, "read")
+                        else _json_result
+                    )
+                )
+            )
+            logging_info(
+                f"ALL DONE in call_cleverthis_api id:{message.id}, resp code:{success_code}"
+            )
+            return DataResponseFactory.of(
+                message.id,
+                success_code,
+                _content_type,
+                _binary_result,
+                message.trace_info,
+            )
+        except HTTPException as http_error:
+            logging_error(f"Service REST endpoint invocation failed: {http_error}")
+            _content = str(http_error.detail)
+            if not is_likely_json(_content):
+                return DataResponseFactory.of_error_message(
+                    message.id,
+                    AMQErrorMessage(
+                        http_error.status_code,
+                        "Service Invocation Failed",
+                        _content,
+                    ),
+                    message.trace_info,
+                )
+            return DataResponseFactory.of(
+                message.id,
+                http_error.status_code,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info,
+            )
+        except Exception as error:
+            logging_error(f"Service endpoint invocation failed: {error}")
+            _content = str(error)
+            if not is_likely_json(_content):
+                return DataResponseFactory.of_error_message(
+                    message.id,
+                    AMQErrorMessage(
+                        500,
+                        "Service Invocation Failed",
+                        _content,
+                    ),
+                    message.trace_info,
+                )
+            return DataResponseFactory.of(
+                message.id,
+                500,
+                "application/json",
+                _content.encode("utf-8"),
+                message.trace_info,
+            )

amqp/adapter/data_message_factory.py

@@ -1,16 +1,28 @@
+"""
+methods to create, serialize and deserialize DataMessage objects.
+"""
+
 import base64
+import json
 import re
 from io import BytesIO
 from datetime import datetime, timezone
 from typing import Dict, List
 
-from amqp.model.model import AMQMessage
+import amqp.adapter.file_handler
+from amqp.model.model import DataMessage
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import map_with_list_as_string, map_as_string, parse_map_list_string, parse_map_string
+from amqp.adapter.serializer import (
+    map_with_list_as_string,
+    map_as_string,
+    parse_map_list_string,
+    parse_map_string,
+)
 from amqp.router.utils import sanitize_as_rabbitmq_name
+from amqp.adapter.file_handler import StreamingFileHandler, FileHandler
 
 
-class AMQMessageFactory:
+class DataMessageFactory:
     SNOWFLAKE_ID_BITS = 80
     SEQUENCE_BITS = 12
     MACHINE_ID_BITS = 24
@@ -36,7 +48,7 @@ class AMQMessageFactory:
         :return: factory instance
         """
         if cls._instance is None:
-            cls._instance = AMQMessageFactory(machine_id)
+            cls._instance = DataMessageFactory(machine_id)
         return cls._instance
 
     def generate_snowflake_id(self) -> SnowflakeId:
@@ -59,20 +71,25 @@ class AMQMessageFactory:
             self.snowflake_sequence = 0
             self.snowflake_last_timestamp = timestamp
 
-        time = (timestamp - 1000 * int(AMQMessageFactory.snowflake_epoch))
+        time = timestamp - 1000 * int(DataMessageFactory.snowflake_epoch)
         return SnowflakeId(
             time >> (64 - self.SEQUENCE_BITS - self.MACHINE_ID_BITS),
-            ((time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS)) | self.machine_id | self.snowflake_sequence) & 0xFFFFFFFFFFFFFFFF
+            (
+                (time << (self.SEQUENCE_BITS + self.MACHINE_ID_BITS))
+                | self.machine_id
+                | self.snowflake_sequence
+            )
+            & 0xFFFFFFFFFFFFFFFF,
         )
 
     def create_request_message(
-            self,
-            method: str,
-            domain: str,
-            path: str,
-            headers: Dict[str, List[str]],
-            message,
-    ) -> AMQMessage:
+        self,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        message,
+    ) -> DataMessage:
         """
         Encapsulate / hide the system level inputs, create the AMQ message from business relevant input
         :param method: a method requested (e.g. HTTP method like GET, POST in case of REST request)
@@ -80,12 +97,14 @@ class AMQMessageFactory:
         :param path: path at the service
         :param headers: HTTP headers
         :param message: payload
-        :return: AMQMessage object
+        :return: DataMessage object
         """
         serializable_headers = headers.copy() if headers else {}
-        payload = base64.b64encode(message.encode('utf-8') if isinstance(message, str) else message)
+        payload = base64.b64encode(
+            message.encode("utf-8") if isinstance(message, str) else message
+        )
         trace_info = {}
-        return AMQMessage(
+        return DataMessage(
             self.MAGIC_BYTE_HTTP_REQUEST,
             self.generate_snowflake_id(),
             method,
@@ -97,10 +116,10 @@ class AMQMessageFactory:
         )
 
     @staticmethod
-    def amq_message_routing_key(message: AMQMessage) -> str:
+    def amq_message_routing_key(message: DataMessage) -> str:
         """
         Constructs the message specific routing key.
-        :param message: AMQMessage input
+        :param message: DataMessage input
         :return: routing key (as string) compliant to RabbitMQ allowed character set.
         """
         return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
@@ -113,19 +132,23 @@ class AMQMessageFactory:
         :return: the message timestamp
         """
         bits = _id.get_bits()
-        low = bits[0] >> (AMQMessageFactory.SEQUENCE_BITS + AMQMessageFactory.MACHINE_ID_BITS)
-        high = bits[1] << (64 - AMQMessageFactory.SEQUENCE_BITS - AMQMessageFactory.MACHINE_ID_BITS)
-        return (high | low) + AMQMessageFactory.snowflake_epoch * 1000
+        low = bits[0] >> (
+            DataMessageFactory.SEQUENCE_BITS + DataMessageFactory.MACHINE_ID_BITS
+        )
+        high = bits[1] << (
+            64 - DataMessageFactory.SEQUENCE_BITS - DataMessageFactory.MACHINE_ID_BITS
+        )
+        return (high | low) + DataMessageFactory.snowflake_epoch * 1000
 
     @staticmethod
-    def to_string(message: AMQMessage) -> str:
+    def to_string(message: DataMessage) -> str:
         """
         Human-readable representation.
-        :param message: AMQMessage input
+        :param message: DataMessage input
         :return: as string
         """
         return (
-            f"AMQMessage{{"
+            f"DataMessage{{"
             f"id={message.id}, "
             f"method='{message.method}', "
             f"domain='{message.domain}', "
@@ -137,39 +160,39 @@ class AMQMessageFactory:
         )
 
     @staticmethod
-    def serialize(message: AMQMessage) -> bytes:
+    def serialize(message: DataMessage) -> bytes:
         """
-        Converts AMQMessage to byte array
+        Converts DataMessage to byte array
         :param message: message to serialize
         :return: messages as bytes
         """
         id_bytes = str(message.id).encode()
         prolog = (
             f"|m={message.method}|d={message.domain}|p={message.path}|h={{{map_with_list_as_string(message.headers)}}}|t={{{map_as_string(message.trace_info)}}}|b="
-        ).encode('utf-8')
+        ).encode("utf-8")
         header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
+        prolog_len = header_length.to_bytes(2, byteorder="big")
         msg_length = 2 + header_length + len(message.base64_body)
         with BytesIO(bytearray(msg_length)) as baos:
             baos.write(prolog_len)
-            baos.write(AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
+            baos.write(DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST.encode("utf-8"))
             baos.write(id_bytes)
             baos.write(prolog)
             baos.write(message.base64_body)
             return baos.getvalue()
 
     @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQMessage:
+    def from_bytes(input_bytes: bytes) -> DataMessage:
         """
-        Builds the AMQMessage from its serialized (byte array) form.
+        Builds the DataMessage from its serialized (byte array) form.
         :param input_bytes: serialized message
-        :return: AMQMessage instance
+        :return: DataMessage instance
         """
         prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
+        metadata = input_bytes[2 : prolog_len + 2].decode()
 
         pattern = re.compile(
-            f"{AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST}"
+            f"{DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST}"
             r"([0-9A-Fa-f]+)\|"
             r"m=([^|]*)\|"
             r"d=([^|]*)\|"
@@ -181,7 +204,7 @@ class AMQMessageFactory:
         match = pattern.match(metadata)
         if not match:
             raise ValueError(
-                f"AMQMessage format does not match expected format. "
+                f"DataMessage format does not match expected format. "
                 f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
             )
 
@@ -191,13 +214,13 @@ class AMQMessageFactory:
         path = match.group(4)
         headers_str = match.group(5)
         trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
+        body_b64 = input_bytes[prolog_len + 2 :]
 
         headers = parse_map_list_string(headers_str)
         trace_info = parse_map_string(trace_info_str)
 
-        return AMQMessage(
-            AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST,
+        return DataMessage(
+            DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST,
             SnowflakeId.from_hex(id_str),
             method,
             domain,
@@ -206,3 +229,29 @@ class AMQMessageFactory:
             trace_info,
             body_b64,
         )
+
+    @staticmethod
+    async def from_stream(
+        stream: bytes,
+        rabbitmq_url: str,
+        loop,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ) -> DataMessage | None:
+        """
+        Builds the DataMessage from its serialized (byte array) form.
+        :param stream: serialized message
+        :param rabbitmq_url: RabbitMQ connection URL
+        :param loop: event loop
+        :param file_handler: file handler for downloading the message
+        :return: DataMessage instance
+        """
+        # async implementation
+        _message_data = json.loads(stream.decode())
+        _req_info = _message_data.get("req-info", "")
+        if _req_info:
+            (body, eof) = await file_handler.download_buffer(
+                loop=loop, rabbitmq_url=rabbitmq_url, queue_name=_req_info
+            )
+            if eof == amqp.adapter.file_handler.END_OF_FILE:
+                return DataMessageFactory.from_bytes(body)
+        return None

amqp/adapter/data_parser.py

@@ -0,0 +1,206 @@
+import io
+import json
+import python_multipart
+from tempfile import SpooledTemporaryFile
+
+from fastapi import UploadFile
+from python_multipart.multipart import Field, File
+from typing import Tuple, Dict
+from urllib.parse import parse_qs, urlparse
+from xml.etree import ElementTree
+
+from amqp.adapter.logging_utils import logging_error, logging_debug
+from amqp.model.model import DataMessage
+
+
+class MultipartFormDataParser:
+    """
+    Parses multipart/form-data content and extracts form fields (key/value pairs) and files.
+    Note: multipart message that contains file(s) to upload are converted to CleverMicro JSON-based streaming format,
+    where files are transmitted separately via stream using dedicated queue, so the message content is no longer
+    the HTTP multipart format, but the Content-Type is being preserved for compatibility with the original message.
+    This class supports both formats (multipart and JSON) depending on the message content, because if no file is
+    included, the message is transmitted in the unchanged multipart/form-data format.
+    """
+
+    def __init__(self, message: DataMessage):
+        self.form_data: dict = {}
+        self.is_multipart = False
+        self.is_json = False
+        self.is_valid = True
+        # Convert each list of strings to a single string joined by newline, then to bytes
+        _headers = {
+            key: "\n".join(value).encode("utf-8")
+            for key, value in message.headers.items()
+        }
+        try:
+            if (
+                hasattr(message, "extra_data")
+                and isinstance(message.extra_data, dict)
+                and len(message.extra_data) > 0
+            ):
+                try:
+                    self.form_data = json.loads(message.body())
+                    self.is_json = True
+                    if self.form_data["files"] and self.form_data["form"]:
+                        self.form_data = AMQDataParser.process_form_data(self.form_data)
+                except Exception as jsonerror:
+                    logging_error(f"Parsing JSON: {jsonerror}")
+                    self.is_valid = False
+            else:
+                python_multipart.parse_form(
+                    _headers, io.BytesIO(message.body()), self.on_field, self.on_file
+                )
+                self.is_multipart = True
+        except Exception as e:
+            logging_error(
+                f"Error parsing multipart/form-data: {e}. Trying parsing as JSON."
+            )
+            try:
+                self.form_data = json.loads(message.body())
+                self.is_json = True
+                if self.form_data["files"] and self.form_data["form"]:
+                    self.form_data = AMQDataParser.process_form_data(self.form_data)
+            except Exception as jsonerror:
+                logging_error(f"Parsing JSON: {jsonerror}")
+                self.is_valid = False
+
+    def on_field(self, field: Field):
+        logging_debug(str(field))
+        self.form_data[field.field_name.decode("utf-8")] = field.value
+
+    def on_file(self, file: File):
+        logging_debug(str(file))
+        file.file_object.seek(0)
+        _spooled_file: SpooledTemporaryFile = SpooledTemporaryFile()
+        _spooled_file.write(file.file_object.read())
+        _spooled_file.seek(0)
+        self.form_data[file.field_name.decode("utf-8")] = UploadFile(
+            _spooled_file, size=file.size, filename=file.file_name.decode("utf-8")
+        )
+
+
+class AMQDataParser:
+    """
+    A class for parsing AMQ data messages. It provides methods to parse form data,
+    """
+
+    @staticmethod
+    def process_form_data(json_input) -> dict:
+        """
+        Processes the input JSON to extract form data. The JSON has two top entries: 'files' and 'form'.
+        This is part of the file stream implementation in AMQ adapter to stream large files in chunks.
+        'files' contains information about the AMQ queue from which read the file, while 'form' contains
+        additional non-file form values.
+        This function combines the file and values at the top level in the returned dictionary.
+        Args:
+            json_input: JSON string or dict containing the form and files data
+
+        Returns:
+            dict: Processed form data with files data overriding non-empty values
+        """
+        # Load JSON if input is string
+        if isinstance(json_input, str):
+            data = json.loads(json_input)
+        else:
+            data = json_input
+
+        result = {}
+
+        # Process form data - take first element of each list
+        for key, value in data["form"].items():
+            if value:  # Only process if list is not empty
+                result[key] = value[0]
+
+        # Override with files data where available
+        for key, value in data["files"].items():
+            if value:  # Only override if files data is non-empty
+                result[key] = value[0]
+
+        return result
+
+    @staticmethod
+    def parse_get_url(url: str) -> Tuple[str, Dict[str, str]]:
+        """
+        Parses an HTTP GET URL and returns (context_path, query_params_dict)
+
+        Args:
+            url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+        Returns:
+            tuple: (context_path, query_params_dict)
+            - context_path: The path part of the URL (e.g., "/path")
+            - query_params_dict: Query parameters where values are str for single value or list of str for multivalue
+        """
+        _parsed = urlparse(url)
+        _query_params = parse_qs(_parsed.query)
+        # Convert values from lists to single values when there's only one value
+        _simplified_params = {
+            k: v[0] if len(v) == 1 else v for k, v in _query_params.items()
+        }
+        return _parsed.path, _simplified_params
+
+    @staticmethod
+    def parse_request_body(message: DataMessage):
+        """
+        Parses the HTTP POST request body based on the MIME type.
+        This is typically called fom the concrete CleverThis Service to extract parameters from the message body.
+        Args:
+            message (DataMessage): Contains raw request and MIME headers.
+
+        Returns:
+            dict: Parsed data as a dictionary.
+        """
+        _content_type = message.headers.get("Content-Type", "")
+        if not _content_type:
+            raise ValueError("Content-Type header is required")
+        content_type = _content_type[0]
+        # Parse JSON
+        if content_type == "application/json":
+            try:
+                return json.loads(message.body())
+            except json.JSONDecodeError as e:
+                logging_error(f"Invalid JSON: {e}")
+                raise ValueError(f"Invalid JSON: {e}")
+
+        # Parse URL-encoded form data
+        elif content_type == "application/x-www-form-urlencoded":
+            try:
+                _form_data: dict = {
+                    k: v[0] if len(v) == 1 else v
+                    for k, v in parse_qs(message.body()).items()
+                }
+                if len(_form_data) == 0 and len(message.body()) > 0:
+                    _form_data = json.loads(message.body())
+                    if _form_data["files"] and _form_data["form"]:
+                        _form_data = MultipartFormDataParser.process_form_data(
+                            _form_data
+                        )
+                return _form_data
+            except Exception as e:
+                logging_error(f"Invalid URL-encoded form data: {e}")
+                raise ValueError(f"Invalid URL-encoded form data: {e}")
+
+        # Parse multipart/form-data
+        elif content_type.startswith("multipart/form-data"):
+            try:
+                parser: MultipartFormDataParser = MultipartFormDataParser(message)
+                return parser.form_data
+            except Exception as e:
+                logging_error(f"Invalid multipart/form-data: {e}")
+                raise ValueError(f"Invalid multipart/form-data: {e}")
+
+        # Parse plain text
+        elif content_type == "text/plain":
+            return {"text": message.body()}
+
+        # Parse XML
+        elif content_type == "application/xml":
+            try:
+                _root = ElementTree.fromstring(message.body())
+                return {elem.tag: elem.text for elem in _root}
+            except ElementTree.ParseError as e:
+                raise ValueError(f"Invalid XML: {e}")
+
+        else:
+            raise ValueError(f"Unsupported Content-Type: {content_type}")

amqp/adapter/data_response_factory.py

@@ -1,80 +1,108 @@
+"""
+method to create, serialize and deserialize DataResponse message
+"""
+
 import base64
 import re
 from io import BytesIO
-from amqp.model.model import AMQResponse
+from amqp.model.model import DataResponse, AMQErrorMessage
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
+from amqp.adapter.serializer import parse_map_string, map_as_string
 
 
-class AMQResponseFactory:
+class DataResponseFactory:
     MAGIC_BYTE_HTTP_REQUEST = "A"
 
     @staticmethod
-    def create_async_response_message(_id: SnowflakeId) -> AMQResponse:
+    def create_async_response_message(_id: SnowflakeId) -> DataResponse:
         """
         Helper method to create standard OK response.
         :param _id: ID
         :return: Response message
         """
-        return AMQResponse(
+        return DataResponse(
             str(_id), 200, "application/text", "OK".encode("utf-8"), "", "", {}
         )
 
     @staticmethod
-    def of(id: SnowflakeId, response_code: int, content_type: str, body: bytes, trace_info: dict[str,str]) -> AMQResponse:
+    def of(
+        id: SnowflakeId,
+        response_code: int,
+        content_type: str,
+        body: bytes,
+        trace_info: dict[str, str],
+        error: str | None = None,
+        error_cause: str | None = None,
+    ) -> DataResponse:
         """
-        Create the AMQResponse message supplying all key values
+        Create the DataResponse message supplying all key values
         :param id: SnowflakeID
         :param response_code: HTTP response code
         :param content_type: MIME content type
         :param body: payload
         :param trace_info: Jaeger trace info (id)
-        :return: AMQResponseMessage object
+        :return: DataResponseMessage object
         """
-        return AMQResponse(
+        return DataResponse(
             id=id.as_string(),
             response_code=response_code,
             content_type=content_type,
             response=body,
-            error=None,
-            error_cause=None,
-            trace_info=trace_info
+            error=error,
+            error_cause=error_cause,
+            trace_info=trace_info,
         )
 
     @staticmethod
-    def serialize(response: AMQResponse) -> bytes:
+    def of_error_message(
+        id: SnowflakeId,
+        error_message: AMQErrorMessage,
+        trace_info: dict[str, str],
+    ) -> DataResponse:
+        return DataResponseFactory.of(
+            id,
+            error_message.response_code,
+            error_message.content_type,
+            error_message.serialize(),
+            trace_info,
+            error=error_message.error,
+            error_cause=error_message.detail,
+        )
+
+    @staticmethod
+    def serialize(response: DataResponse) -> bytes:
         """
         Serialize the message to byte array (to be sent over RabbitMQ)
         :param response: Object to serialize
         :return: byte array as serialized message
         """
-        id_bytes = response.id.encode('utf-8')
+        id_bytes = response.id.encode("utf-8")
         prolog = (
             f"|r={response.response_code}|c={response.content_type}|e={response.error}|f={response.error_cause}|t={{{map_as_string(response.trace_info)}}}|b="
-        ).encode('utf-8')
+        ).encode("utf-8")
         header_length = 1 + len(id_bytes) + len(prolog)
-        prolog_len = header_length.to_bytes(2, byteorder='big')
+        prolog_len = header_length.to_bytes(2, byteorder="big")
         msg_length = 2 + header_length + len(response.response)
         with BytesIO(bytearray(msg_length)) as baos:
             baos.write(prolog_len)
-            baos.write(AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST.encode('utf-8'))
+            baos.write(DataResponseFactory.MAGIC_BYTE_HTTP_REQUEST.encode("utf-8"))
             baos.write(id_bytes)
             baos.write(prolog)
             baos.write(base64.b64encode(response.response))
             return baos.getvalue()
 
     @staticmethod
-    def from_bytes(input_bytes: bytes) -> AMQResponse:
+    def from_bytes(input_bytes: bytes) -> DataResponse:
         """
-        Buil;ds the AMQResponse object from its serialized form
+        Buil;ds the DataResponse object from its serialized form
         :param input_bytes: byte array - serialized response
-        :return: AMQResponse object
+        :return: DataResponse object
         """
         prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-        metadata = input_bytes[2:prolog_len + 2].decode()
+        metadata = input_bytes[2 : prolog_len + 2].decode()
 
         pattern = re.compile(
-            f"{AMQResponseFactory.MAGIC_BYTE_HTTP_REQUEST}"
+            f"{DataResponseFactory.MAGIC_BYTE_HTTP_REQUEST}"
             r"([0-9A-Fa-f]+)\|"
             r"r=([^|]*)\|"
             r"c=([^|]*)\|"
@@ -86,7 +114,7 @@ class AMQResponseFactory:
         match = pattern.match(metadata)
         if not match:
             raise ValueError(
-                f"AMQResponse format does not match expected format. "
+                f"DataResponse format does not match expected format. "
                 f"Input string [{metadata}] does not match pattern: {pattern.pattern}"
             )
 
@@ -96,11 +124,11 @@ class AMQResponseFactory:
         error = match.group(4)
         error_cause = match.group(5)
         trace_info_str = match.group(6)
-        body_b64 = input_bytes[prolog_len + 2:]
+        body_b64 = input_bytes[prolog_len + 2 :]
 
         trace_info = parse_map_string(trace_info_str)
 
-        return AMQResponse(
+        return DataResponse(
             id_str,
             response_code,
             content_type,
@@ -109,3 +137,7 @@ class AMQResponseFactory:
             error_cause,
             trace_info,
         )
+
+
+# Ensure backward compatibility
+AMQResponseFactory = DataResponseFactory

amqp/adapter/file_handler.py

@@ -0,0 +1,407 @@
+"""
+implements the file downloader for the amqp adapter, where file is sent in chunks over RabbitMQ
+"""
+
+import asyncio
+import json
+import logging
+import os
+from asyncio import AbstractEventLoop
+from collections import defaultdict
+from pathlib import Path
+
+from aio_pika import Message, connect_robust
+from aio_pika.abc import (
+    AbstractExchange,
+    AbstractIncomingMessage,
+    AbstractQueue,
+    AbstractRobustChannel,
+    DeliveryMode,
+)
+
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.router.utils import await_result
+
+# Global dictionary to track completed downloads
+download_locations = defaultdict(str)
+IN_PROGRESS = 0
+END_OF_FILE = 1
+CANCELLED = 2
+
+
+class FileHandler:
+    """
+    The abstract base class for file handling. This is mainly to make the code testable, as there are only 2
+    options for pushing (uploading) a file(s) to a service: as HTTP multipart message delivered via RabbitMQ
+    or via stream in RabbitMQ. Downloading a file is always done via RabbitMQ stream. Thus the method signiture
+    is tighly coupled to the RabbitMQ API in v1. Only alternative implementation is a mock object in unittests.
+    """
+
+    def __init__(self):
+        pass
+
+    def ensure_directory_exists(self, filepath) -> str:
+        """
+        Ensures that the directory for the given filepath exists, and handles file versioning.
+
+        Args:
+            filepath: The path to the file.
+
+        Returns:
+            The original filepath if the file does not exist, or a modified filepath
+            pointing to the next available version of the file if it does.
+        """
+        # 1. Ensure directory exists
+        directory = os.path.dirname(filepath)
+        if directory and not os.path.exists(directory):
+            os.makedirs(directory)
+
+        # 2. Handle file versioning
+        if not os.path.exists(filepath):
+            return filepath  # File doesn't exist, return original path
+
+        # File exists, find next available version
+        base, ext = os.path.splitext(filepath)
+        version = 1
+        while True:
+            new_filepath = f"{base}.{version}{ext}"
+            if not os.path.exists(new_filepath):
+                return new_filepath  # Found a free version, return this path
+            version += 1
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearray from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file
+        to be downloaded. File may consist of multiple chunks, so the function will wait until all chunks are received.
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            rabbitmq_url: The RabbitMQ connection URL.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+# The StreamingFileHandler class is a subclass of FileHandler that implements the download_file method
+class StreamingFileHandler(FileHandler):
+    """
+    Handles file downloads from RabbitMQ streams.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+    async def download_file(
+        self, loop, rabbitmq_url, file_info, message_data, output_dir
+    ) -> (int, int):
+        """
+        Asynchronously downloads a file from RabbitMQ based on the provided file information.
+        takes the file_info and message_data and output_dir, creates queue consumer and waits for the file to be downloaded
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            file_info (dict): A dictionary containing file metadata
+            message_data (dict): The entire message
+            output_dir (str): The directory where the file should be saved.
+        """
+        filename = file_info["filename"]
+        size = file_info["size"]
+        stream_name = file_info["streamName"]
+        queue_name = file_info["queue_name"]
+        filepath = os.path.join(output_dir, filename)
+        filepath = self.ensure_directory_exists(
+            filepath
+        )  # if filename exists, create a new (next) version
+        download_locations[file_info["key"]] = filepath
+        filename = os.path.basename(
+            filepath
+        )  # if a new version was created, ensure to use that new version
+        file_info["filename"] = filename
+
+        logging_debug(
+            f"Downloading file: {filename} (size: {size} bytes) from stream: {stream_name}:{queue_name} to {filepath}"
+        )
+
+        async def wrap_rabbit_mq_calls() -> tuple[int, int] | None:
+            connection = await connect_robust(rabbitmq_url, loop=loop)
+            try:
+                _file_size = 0
+                _eof = IN_PROGRESS
+                channel: AbstractRobustChannel = await connection.channel()
+                queue: AbstractQueue = await channel.get_queue(queue_name, ensure=False)
+                logging_debug(
+                    f"Awaiting all file chunks being downloaded for {filename}"
+                )
+                async with queue.iterator() as queue_iter:
+                    async for message in queue_iter:
+                        async with message.process():
+                            with open(filepath, "ab") as f:
+                                f.write(message.body)
+                            _file_size = _file_size + message.body_size
+                            _eof = int(message.headers.get("eof", END_OF_FILE))
+                            if _eof != IN_PROGRESS:
+                                break
+            except Exception as e:
+                logging_error(f"File download failed: {e}")
+            finally:
+                await connection.close()
+
+            return _file_size, _eof
+
+        if queue_name:
+            _local_file_size, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+            )
+            return _local_file_size, _local_eof
+
+        return 0, CANCELLED
+
+    async def download_buffer(self, loop, rabbitmq_url, queue_name) -> (bytearray, int):
+        """
+        Asynchronously downloads a bytearay from RabbitMQ based on the provided file information.
+        It assumes entire array is in the first (and only) chunk
+        Args:
+            loop: The asyncio event loop.
+            rabbitmq_url: The RabbitMQ connection URL.
+            queue_name (str): The directory where the file should be saved.
+        """
+        logging_debug(f"Downloading buffer from stream: {queue_name}")
+        if queue_name:
+
+            async def wrap_rabbit_mq_calls() -> tuple[bytearray, int]:
+                _connection = await connect_robust(rabbitmq_url, loop=loop)
+                _eof = IN_PROGRESS
+                _res = bytearray()
+                try:
+                    channel: AbstractRobustChannel = await _connection.channel()
+                    queue: AbstractQueue = await channel.get_queue(
+                        queue_name, ensure=False
+                    )
+                    logging_debug(
+                        f"Awaiting all buffer chunks being downloaded for {queue_name}"
+                    )
+                    async with queue.iterator() as queue_iter:
+                        async for message in queue_iter:
+                            async with message.process():
+                                _res.extend(message.body)
+                                _eof = message.headers.get("eof", END_OF_FILE)
+                                if _eof != IN_PROGRESS:
+                                    break
+                    logging_info(f"Finished downloading buffer {queue_name}")
+                except Exception as e:
+                    logging_error(f"Downloading buffer: {e}")
+
+                await _connection.close()
+                return _res, _eof
+
+            _local_res, _local_eof = await await_result(
+                asyncio.run_coroutine_threadsafe(wrap_rabbit_mq_calls(), loop=loop)
+            )
+            return _local_res, _local_eof
+        return 0, CANCELLED
+
+    async def on_message(
+        self,
+        message: AbstractIncomingMessage,
+        json_data: bytes,
+        rabbitmq_url: str,
+        loop,
+        output_dir="downloaded_files",
+    ) -> defaultdict:
+        """
+        Asynchronous callback function for handling messages from the 'amq-cleverswarm' queue.
+
+        Args:
+            message: The incoming RabbitMQ message.
+            json_data: The JSON data from the message.
+            rabbitmq_url: The RabbitMQ connection URL.
+            loop: The asyncio event loop of the RabbitMQ API
+            output_dir
+        """
+        try:
+            _message_data = json.loads(message.body.decode())
+            _amq_message_data = json.loads(json_data.decode())
+
+            files_to_download = []
+            for _key, _file_type in _amq_message_data["files"].items():
+                if len(_file_type) > 0:
+                    files_info = _file_type[0]
+                    stream_name = files_info["streamName"]
+                    files_info["queue_name"] = (
+                        _message_data.get(stream_name, "") if stream_name else ""
+                    )
+                    files_info["key"] = _key
+                    files_to_download.append(files_info)
+
+            if not files_to_download:
+                logging_info(
+                    " [%s] No files to download in this message.", message.delivery_tag
+                )
+                # await message.ack()
+                return defaultdict()
+
+            # Create a task for each file download
+            _tasks = []
+            for file_info in files_to_download:
+                logging_info(
+                    f" [{message.delivery_tag}] about to create task for {file_info}"
+                )
+                _task = asyncio.create_task(
+                    self.download_file(
+                        loop, rabbitmq_url, file_info, _message_data, output_dir
+                    )
+                )
+                _tasks.append(_task)
+            logging_info(
+                f" [{message.delivery_tag}] Waiting for all files to download...gather(*tasks)"
+            )
+            await asyncio.gather(*_tasks)  # Wait for all downloads to complete
+
+            logging_info(f"All files downloaded. d-tag: {message.delivery_tag}")
+        except Exception as e:
+            logging_error(f"Building DataMessage: {e}")
+            asyncio.run_coroutine_threadsafe(
+                message.ack(), loop=loop
+            )  # ACK -> ignore the message (do not redeliver)
+
+        return download_locations
+
+    async def publish_file(
+        self,
+        exchange: AbstractExchange,
+        file_path: Path,
+        routing_key: str,
+        loop: AbstractEventLoop,
+        max_chunk_size: int = 2**20,  # 1MB default chunk size
+        content_type: str = "application/octet-stream",  # Default content type
+        delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
+    ) -> tuple[str, int]:
+        """
+        Reads a file from the file system and publishes its content to a RabbitMQ exchange
+        in chunks.  It declares a queue with a random name, binds it to the specified
+        exchange, and uses the queue name as the routing key.
+
+        Args:
+            exchange: The aio_pika RobustChannel to use for communication with RabbitMQ.
+            file_path: The path to the file to publish.
+            max_chunk_size: The maximum size of each chunk (in bytes).
+            routing_key: The name of the RabbitMQ exchange routing key to publish to correct queue.
+            loop: correct event loop
+            max_chunk_size: chunk size to send
+            content_type: The content type of the file data.
+            delivery_mode: The delivery mode (e.g., PERSISTENT or TRANSIENT).
+
+        Returns:
+            The name of the randomly generated queue where the file content was published.
+        """
+        if not file_path.is_file():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        _file_size = os.path.getsize(file_path)
+
+        with open(file_path, "rb") as f:
+            _offset = 0
+            while True:
+                _chunk = f.read(max_chunk_size)
+                if not _chunk:
+                    break  # End of file
+
+                message = Message(
+                    body=_chunk,
+                    content_type=content_type,
+                    delivery_mode=delivery_mode,
+                    headers={
+                        "file_name": file_path.name,
+                        "pos": _offset,
+                        "total_size": _file_size,
+                        "chunk_size": len(_chunk),
+                        "eof": (
+                            END_OF_FILE
+                            if max_chunk_size + _offset >= _file_size
+                            else IN_PROGRESS
+                        ),
+                    },
+                )
+                logging.debug(
+                    f"Going to Publish chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+                _future = asyncio.run_coroutine_threadsafe(
+                    exchange.publish(message, routing_key=routing_key), loop
+                )
+                while not _future.done():
+                    # sleep allows coroutine to execute. must wait for completion to ensure all chunks written in right order
+                    await asyncio.sleep(0.02)
+                    logging_debug(
+                        f"Waiting for chunk {_offset}/{_file_size} future= {_future}"
+                    )
+
+                _offset += len(_chunk)
+                logging_debug(
+                    f"Published chunk {_offset}/{_file_size} to {exchange.name}:{routing_key}"
+                )
+
+        logging_debug(
+            f"File {file_path} published to RabbitMQ exchange {exchange.name}, queue {routing_key}"
+        )
+        return routing_key, _file_size

amqp/adapter/logging_utils.py

@@ -0,0 +1,128 @@
+import inspect
+import logging
+import os
+import sys
+import threading
+import traceback
+from importlib.metadata import version
+
+# Get version from installed package metadata
+__version__ = version("amq_adapter")  # Name must match pyproject.toml [project] name
+__verbose__ = os.environ.get("AMQ_VERBOSE_LOGGING", "0") == "1"
+
+
+def get_context_info():
+    """
+    Retrieves the current module, line number, and function name.
+
+    Returns:
+        A tuple containing (module name, line number, function name).
+        Returns (None, None, None) if it fails to retrieve the information.
+    """
+    try:
+        # Get the frame object for the current function call
+        frame = inspect.currentframe()
+        if frame is not None:
+            # Get the frame object for the caller of the current function
+            frame = frame.f_back.f_back
+        if frame:
+            code_context = frame.f_code
+            module_name = code_context.co_filename
+            module_name = (
+                "amqp" + module_name.split("/amqp", 1)[-1]
+                if "/amqp" in module_name
+                else module_name
+            )
+            line_number = frame.f_lineno
+            function_name = code_context.co_name
+            return module_name, line_number, function_name
+        else:
+            return None, None, None
+    except Exception:
+        return None, None, None
+
+
+def logging_error(msg: str, *args) -> None:
+    """
+    Log an error message according to current logging for 'ERROR' level.
+    Add module name, line and function name where the error occurred, on single line.
+    Args:
+        msg (str): The error message to log.
+    """
+    _exec_info = sys.exc_info()[2]
+    _thread_id = threading.get_ident()
+    if _exec_info:
+        _tb = traceback.extract_tb(sys.exc_info()[2])[-1]
+        logging.error(
+            f"{_thread_id}:{__version__} [E] '{msg}' ---> '{_tb.filename}':{_tb.lineno}, '{_tb.name}()'",
+            *args,
+        )
+    else:
+        module, line_number, function_name = get_context_info()
+        if module and line_number and function_name:
+            logging.error(
+                f"{_thread_id}:{__version__} [E] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+                *args,
+            )
+        else:
+            logging.error(f"{_thread_id}:{__version__} [E] {msg}", *args)
+
+
+def logging_warning(msg: str, *args) -> None:
+    """
+    Log a warning message according to current logging for 'WARN' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The warning message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not __verbose__:
+        logging.warning(f"{_thread_id}:{__version__} [W] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.warning(
+            f"{_thread_id}:{__version__} [W] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.warning(f"{_thread_id}:{__version__} [W] {msg}", *args)
+
+
+def logging_info(msg: str, *args) -> None:
+    """
+    Log an info message according to current logging for 'INFO' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The info message to log.
+    """
+    _thread_id = threading.get_ident()
+    if not __verbose__:
+        logging.info(f"{_thread_id}:{__version__} [*] {msg}", *args)
+        return
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.info(
+            f"{_thread_id}:{__version__} [*] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.info(f"{_thread_id}:{__version__} [*] {msg}", *args)
+
+
+def logging_debug(msg: str, *args) -> None:
+    """
+    Log a debug message according to current logging for 'DEBUG' level.
+    Add module name, line and function name where the print occurred, on single line.
+    Args:
+        msg (str): The debug message to log.
+    """
+    _thread_id = threading.get_ident()
+    module, line_number, function_name = get_context_info()
+    if module and line_number and function_name:
+        logging.info(
+            f"{_thread_id}:{__version__} [DBG] '{msg}' ---> '{module}':{line_number}, '{function_name}()'",
+            *args,
+        )
+    else:
+        logging.debug(f"{_thread_id}:{__version__} [DBG] {msg}", *args)

amqp/adapter/pydantic_serializer.py

@@ -0,0 +1,172 @@
+"""
+This module provides serialization and deserialization functions for Pydantic models that extend BaseModel
+"""
+
+import uuid
+import json
+from datetime import datetime
+from typing import Dict, Any
+from pydantic import BaseModel
+from urllib.parse import urlparse, parse_qs
+
+from amqp.adapter.logging_utils import logging_debug
+
+
+def serialize_object(obj: BaseModel) -> str:
+    """
+    Serializes an R2RSerializable object to a string in the format
+    "<ClassName>:{key1:value1, key2:value2,...}".  Handles UUIDs and datetimes,
+    and nested Pydantic models.
+
+    Args:
+        obj: The BaseModel object to serialize.
+
+    Returns:
+        A string representation of the object.
+    """
+    class_name = obj.__class__.__name__
+    obj_dict = {}
+    for key, value in obj.model_dump().items():
+        if isinstance(value, uuid.UUID):
+            obj_dict[key] = str(value)
+        elif isinstance(value, datetime):
+            obj_dict[key] = value.isoformat()
+        elif isinstance(value, dict):  # Ensure dictionaries are handled
+            obj_dict[key] = value
+        elif isinstance(value, BaseModel):
+            obj_dict[key] = serialize_object(value)  # Recursively serialize
+        elif isinstance(value, list):
+            serialized_list = []
+            for item in value:
+                if isinstance(item, BaseModel):
+                    serialized_list.append(serialize_object(item))
+                else:
+                    serialized_list.append(item)
+            obj_dict[key] = serialized_list
+        else:
+            obj_dict[key] = value
+
+    dict_string = ",".join(f"{k}:{v}" for k, v in obj_dict.items())
+    return f"{class_name}:{{{dict_string}}}"
+
+
+def deserialize_object(data: str) -> BaseModel:
+    """
+    Deserializes a string in the format "<ClassName>:{key1:value1, key2:value2,...}"
+    back into an BaseModel object.  Handles UUIDs, datetimes, and
+    nested Pydantic models.
+
+    Args:
+        data: The string to deserialize.
+
+    Returns:
+        An R2RSerializable object.
+    """
+    class_name, dict_str = data.split(":", 1)
+    dict_str = dict_str.strip("{}")
+    pairs = dict_str.split(",") if dict_str else []
+
+    obj_dict: Dict[str, Any] = {}
+    for pair in pairs:
+        if not pair or ":" not in pair:
+            continue
+        k, v = pair.split(":", 1)
+        v = v.strip()  # important
+        obj_dict[k] = _convert_value(v)
+
+    logging_debug(f"Deserialized object: {class_name}")
+    if "[" in class_name:
+        class_name = class_name.split("[")[0]
+    logging_debug(f"trying object: {class_name}")
+    cls = globals().get(class_name)
+    if cls is None:
+        raise ValueError(f"Unknown class: {class_name}")
+
+    return cls(**obj_dict)
+
+
+def _convert_value(v: str) -> Any:
+    """
+    Helper function to convert a string to its appropriate Python type.
+    """
+    if v == "None":
+        return None
+    elif _is_uuid_string(v):
+        return uuid.UUID(v)
+    elif _is_datetime_string(v):
+        return datetime.fromisoformat(v)
+    elif _is_int_string(v):
+        return int(v)
+    elif (
+        v.startswith("{")
+        and v.endswith("}")
+        and ":" in v
+        and "{" not in v[1:-1]
+        and "}" not in v[1:-1]
+    ):  # nested object
+        # check if it is a json dict
+        try:
+            json.loads(v)
+            return json.loads(v)
+        except json.JSONDecodeError:
+            return deserialize_object(v)  # recursive call
+    elif v.startswith("[") and v.endswith("]"):  # list
+        v = v[1:-1]  # remove brackets
+        items = v.split(",") if v else []
+        deserialized_list = []
+        for item in items:
+            item = item.strip()
+            deserialized_list.append(_convert_value(item))  # recursive call
+        return deserialized_list
+    else:
+        return v  # Keep as string
+
+
+def _is_uuid_string(s: str) -> bool:
+    """Helper to check if a string is a valid UUID."""
+    try:
+        uuid.UUID(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_datetime_string(s: str) -> bool:
+    """Helper to check if a string is a valid datetime in ISO format"""
+    try:
+        datetime.fromisoformat(s)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_int_string(s: str) -> bool:
+    """Helper to check if a string is a valid int."""
+    try:
+        int(s)
+        return True
+    except ValueError:
+        return False
+
+
+def parse_url_query(url):
+    """
+    Parses a HTTP GET URL and returns (context_path, query_params_dict)
+
+    Args:
+        url: The URL string to parse (e.g., "http://example.com/path?foo=1&bar=2&foo=3")
+
+    Returns:
+        tuple: (context_path, query_params_dict)
+        - context_path: The path part of the URL (e.g., "/path")
+        - query_params_dict: Dictionary of query parameters where values are always lists
+                           (e.g., {"foo": ["1", "3"], "bar": ["2"]})
+    """
+    parsed = urlparse(url)
+    query_params = parse_qs(parsed.query)
+
+    # Convert values from lists to single values when there's only one value
+    # If you prefer to always keep values as lists, remove this comprehension
+    simplified_params = {k: v[0] if len(v) == 1 else v for k, v in query_params.items()}
+
+    return (parsed.path, simplified_params)

amqp/adapter/serializer.py

@@ -1,19 +1,23 @@
+"""
+Helper functions used to aid with (de)serialization of messages sent over AMQP.
+"""
+
 import struct
 from io import BytesIO
 from typing import List, Union, Dict
 
 
 def long_to_bytes(x: int) -> bytes:
-    return struct.pack('q', x)
+    return struct.pack("q", x)
 
 
 def bytes_to_long(b: bytes) -> int:
-    return struct.unpack('q', b)[0]
+    return struct.unpack("q", b)[0]
 
 
 def read_long(bis: BytesIO) -> int:
     try:
-        return struct.unpack('q', bis.read(8))[0]
+        return struct.unpack("q", bis.read(8))[0]
     except IOError:
         return 0
 
@@ -25,37 +29,41 @@ def object_or_list_as_string(value: Union[str, List[str]]) -> str:
 
 
 def map_as_string(map_: Dict[str, str]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
 
 
 def map_with_list_as_string(map_: Dict[str, List[str]]) -> str:
-    return ','.join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
+    return ",".join(f"{k}={object_or_list_as_string(v)}" for k, v in map_.items())
 
 
 def parse_map_string(input_str: str) -> Dict[str, str]:
     map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for entry in input_str[1:-1].split(','):
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for entry in input_str[1:-1].split(","):
             if len(entry) > 1:
-                key, value = entry.split('=')
+                key, value = entry.split("=")
                 map_[key] = value
     return map_
 
 
 def csv_as_list(input_str: str) -> List[str]:
-    return [item.strip() for item in input_str.strip('[]').split(',')]
+    return [item.strip() for item in input_str.strip("[]").split(",")]
 
 
 def add_to_map(map_: Dict[str, List[str]], token: str) -> None:
-    eq_idx = token.index('=')
+    eq_idx = token.index("=")
     if eq_idx > 0:
         key = token[:eq_idx]
-        map_[key] = csv_as_list(token[eq_idx+1:])
+        map_[key] = csv_as_list(token[eq_idx + 1 :])
 
 
 def parse_map_list_string(input_str: str) -> Dict[str, List[str]]:
     map_ = {}
-    if input_str.startswith('{') and input_str.endswith('}'):
-        for token in input_str[1:-1].split('],'):
+    if input_str.startswith("{") and input_str.endswith("}"):
+        for token in input_str[1:-1].split("],"):
             add_to_map(map_, token)
     return map_
+
+
+def str_to_bool(value: str) -> bool:
+    return value.lower() in ("true", "yes", "1", "t", "y", "on")

amqp/adapter/service_message_factory.py

@@ -1,86 +1,91 @@
 import base64
-import logging
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import logging_error, logging_info
+from amqp.adapter.serializer import map_as_string, parse_map_string
 from amqp.adapter.trace_info_adapter import TraceInfoAdapter
-from amqp.model.model import CleverMicroServiceMessage
+from amqp.model.model import ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
-from amqp.router.serializer import parse_map_string, map_as_string
 
 RS = "|"
 SPLIT_RS = r"\|"
 
 
-INVALID_MESSAGE = CleverMicroServiceMessage()  # default values mean invalid message
+INVALID_MESSAGE = ServiceMessage()  # default values mean invalid message
 
 
-class CleverMicroServiceMessageFactory:
+class ServiceMessageFactory:
     """
     build/serialize/deserialize CleverMicro Service Message class
     """
+
     def __init__(self, name: str):
         self.process_name = name
 
-    def to_string(self, message: CleverMicroServiceMessage) -> str:
+    def to_string(self, message: ServiceMessage) -> str:
         """
-        Convert CleverMicroServiceMessage to human-readable format
-        :param message: CleverMicroServiceMessage object
+        Convert ServiceMessage to human-readable format
+        :param message: ServiceMessage object
         :return: as string
         """
         if message is not None:
             return (
-                f"CleverMicroServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
+                f"ServiceMessage{{id={message.id.as_string()}|type={self.format_message_type(message)}|"
                 f"recipient={message.recipient_name}|replyTo={message.reply_to}|"
                 f"traceInfo={message.trace_info}|body={base64.b64encode(message.message.encode()).decode() if message.payload_type & 0x80 else message.message}}}"
             )
         return "null"
 
-    def of(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
+    def of(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
         """
-        Builds CleverMicroServiceMessage object from relevant values.
+        Builds ServiceMessage object from relevant values.
         :param message_type: enum Message type
         :param payload: Message content, stored as-is
         :param recipient_name: Recipient name (from the routing expression)
-        :return: CleverMicroServiceMessage  object
+        :return: ServiceMessage  object
         """
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id),
-            message=payload
+            message=payload,
         )
 
-    def as_base64(self, message_type: ServiceMessageType, payload: str, recipient_name: str) -> CleverMicroServiceMessage:
+    def as_base64(
+        self, message_type: ServiceMessageType, payload: str, recipient_name: str
+    ) -> ServiceMessage:
         """
-        Builds CleverMicroServiceMessage object from relevant values, stores content Base64 encoded.
+        Builds ServiceMessage object from relevant values, stores content Base64 encoded.
         :param message_type: enum Message type
         :param payload: Message content, stored as Base64 string
         :param recipient_name: Recipient name (from the routing expression)
-        :return: CleverMicroServiceMessage  object
+        :return: ServiceMessage  object
         """
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        return CleverMicroServiceMessage(
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        return ServiceMessage(
             id=_id,
             payload_type=0x80,
             message_type=message_type,
             recipient_name=recipient_name,
             reply_to=self.process_name,
             trace_info=TraceInfoAdapter.create_produce_span(_id.as_string()),
-            message=payload
+            message=payload,
         )
 
-    def from_bytes(self, input_bytes: bytes) -> CleverMicroServiceMessage:
+    def from_bytes(self, input_bytes: bytes) -> ServiceMessage:
         """
-        Deserialize CleverMicroServiceMessage from byte array serialized form.
-        :param input_bytes: serialized CleverMicroServiceMessage
-        :return: CleverMicroServiceMessage object represented by the input.
+        Deserialize ServiceMessage from byte array serialized form.
+        :param input_bytes: serialized ServiceMessage
+        :return: ServiceMessage object represented by the input.
         """
-        logging.info(f"CleverMicroServiceMessage >>> [{input_bytes.decode()}]")
+        logging_info(f"ServiceMessage >>> [{input_bytes.decode()}]")
         input_str_array = input_bytes.decode().split(RS)
         try:
             i = 0
@@ -93,47 +98,58 @@ class CleverMicroServiceMessageFactory:
             i += 1
             reply_to = input_str_array[i] if i < len(input_str_array) else ""
             i += 1
-            trace_info = parse_map_string(input_str_array[i]) if i < len(input_str_array) else {}
+            trace_info = (
+                parse_map_string(input_str_array[i]) if i < len(input_str_array) else {}
+            )
             i += 1
             message = input_str_array[i] if i < len(input_str_array) else ""
-            return CleverMicroServiceMessage(
+            return ServiceMessage(
                 id=SnowflakeId.from_hex(_id),
                 payload_type=payload_type,
                 message_type=message_type,
                 recipient_name=recipient_name,
                 reply_to=reply_to,
                 trace_info=trace_info,
-                message=base64.b64decode(message).decode() if payload_type & 0x80 else message
+                message=(
+                    base64.b64decode(message).decode()
+                    if payload_type & 0x80
+                    else message
+                ),
             )
         except Exception as e:
-            logging.error(
+            logging_error(
                 "Unexpected ServiceMessage format. Expected 6 tokens separated by '|' in format: "
                 "[ID(64bit number)|MessageType(8bit number)|RecipientName(String)|"
                 "ReplyTo(String)|TraceInfo|Payload(String or Base64)], got: "
-                "[%s], reported problem: %s", input_bytes.decode(), str(e)
+                "[%s], reported problem: %s",
+                input_bytes.decode(),
+                str(e),
             )
             return INVALID_MESSAGE
 
-    def serialize(self, message: CleverMicroServiceMessage) -> bytes:
+    def serialize(self, message: ServiceMessage) -> bytes:
         """
-        Serialize CleverMicroServiceMessage to byte array.
-        :param message: CleverMicroServiceMessage to serialize.
+        Serialize ServiceMessage to byte array.
+        :param message: ServiceMessage to serialize.
         :return: serialized object.
         """
         return (
             f"{message.id.as_string()}{RS}{self.format_message_type(message)}{RS}{message.recipient_name}{RS}"
             f"{message.reply_to}{RS}{map_as_string(message.trace_info)}{RS}"
             f"{base64.b64encode(message.message.encode()).decode() if message.payload_type != 0 else message.message}"
-        ).encode('utf-8')
+        ).encode("utf-8")
 
-
-    def format_message_type(self, message: CleverMicroServiceMessage) -> str:
+    def format_message_type(self, message: ServiceMessage) -> str:
         """
         Formats the Enum message type and decorates it with base64 flag at highest bit.
         For compatibility with 0-based enum ordinals in Java subtract 1 from ordinal value.
         :param message: Service message to send
         :return: formatted message type
         """
-        num_type = (message.message_type.value if message.message_type else ServiceMessageType.UNKNOWN.value) - 1
-        fmt = ((num_type & 0x7F) | (message.payload_type & 0x80))
+        num_type = (
+            message.message_type.value
+            if message.message_type
+            else ServiceMessageType.UNKNOWN.value
+        ) - 1
+        fmt = (num_type & 0x7F) | (message.payload_type & 0x80)
         return f"{fmt:02X}"

amqp/adapter/test_amq_message_factory.py

@@ -1,68 +0,0 @@
-from datetime import datetime, timezone
-from unittest import TestCase
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-
-
-class AMQMessageFactoryTest(TestCase):
-    def test_from_bytes(self):
-        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
-        msg = AMQMessageFactory.from_bytes(raw.encode("utf-8"))
-        self.assertIsNotNone(msg.id)
-        self.assertEqual(msg.id.as_string(),'64F3B5AF4D00000A4000'.lower(),'SnowflakeID incorrectly parsed.')
-        self.assertEqual(msg.headers["Content-Type"][0], "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v")
-
-    def test_generate_snowflake_id(self):
-        _id = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        self.assertEqual('00001000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _id = AMQMessageFactory(2).generate_snowflake_id()
-        self.assertEqual('00002000', _id.as_string()[-8:], 'Instance ID not set correctly')
-
-        _f = AMQMessageFactory(127)
-        _i0 = _f.generate_snowflake_id()
-        _i1 = _f.generate_snowflake_id()
-        _i2 = _f.generate_snowflake_id()
-        _i3 = _f.generate_snowflake_id()
-        self.assertEqual('0007F000'.lower(), _i0.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F001'.lower(), _i1.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F002'.lower(), _i2.as_string()[-8:], 'Instance ID and sequence not set correctly')
-        self.assertEqual('0007F003'.lower(), _i3.as_string()[-8:], 'Instance ID and sequence not set correctly')
-
-    def test_create_request_message(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        self.assertEqual("localhost", _req.domain, 'Domain not set correctly')
-        self.assertEqual("/test/me", _req.path, 'Context path not set correctly')
-        self.assertEqual("v1", _req.headers['k1'][0], "Headers not set correctly")
-        self.assertEqual(b'Quick brown fox', _req.body(), "Body not correctly decoded")
-
-    def test_serialize(self):
-        _f = AMQMessageFactory(34)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _ser = _f.serialize(_req)
-        self.assertEqual(65, _ser[2])
-        _deser = _f.from_bytes(_ser)
-        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
-        self.assertEqual("localhost", _deser.domain)
-        self.assertEqual("/test/me", _deser.path)
-        self.assertEqual(b"Quick brown fox", _deser.body())
-        self.assertEqual("v2", _deser.headers["k2"][0])
-
-    def test_amq_message_routing_key(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _key = _f.amq_message_routing_key(_req)
-        self.assertEqual("localhost.test.me", _key, "Message routing key wrongly extracted from the message")
-
-    def test_get_timestamp_from_id(self):
-        _f = AMQMessageFactory(35)
-        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
-        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
-        self.assertLessEqual(_x_timestamp - _timestamp, 1, "Timestamp in SnowflakeId differs too much from current time")
-
-    def test_to_string(self):
-        _f = AMQMessageFactory(35)
-        _req = _f.create_request_message("PUT", "localhost", "/test/me", {"k1": ["v1"], "k2": ["v2"]}, "Quick brown fox")
-        _as_str = _f.to_string(_req)
-        self.assertGreater(_as_str.index("domain='localhost'"), 0, "Expected content not found")

amqp/adapter/test_amq_route_factory.py

@@ -1,20 +0,0 @@
-from unittest import TestCase
-
-from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute
-
-
-class TestAMQRouteFactory(TestCase):
-    def test_from_string(self):
-        _f = AMQRouteFactory()
-        _r: AMQRoute = _f.from_string("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0")
-        self.assertEqual("", _r.exchange)
-        self.assertEqual("amq-clever-pybook", _r.queue)
-        self.assertEqual(5, _r.timeout)
-        _r2: AMQRoute = _f.from_string("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5")
-        self.assertEqual(NULL_ROUTE, _r2)
-
-    def test_validate(self):
-        _f = AMQRouteFactory()
-        self.assertTrue(_f.validate("amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"))
-        self.assertFalse(_f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#"))

amqp/adapter/trace_info_adapter.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Dict
 
+
 @dataclass
 class TraceInfoAdapter:
     """
@@ -9,6 +10,4 @@ class TraceInfoAdapter:
 
     @staticmethod
     def create_produce_span(message_id: str) -> Dict[str, str]:
-        return {
-            "trace-id": message_id
-        }
+        return {"trace-id": message_id}

amqp/config/amq_configuration.py

@@ -1,8 +1,11 @@
 import configparser
+import inspect
 import os
 
+from amqp.adapter.logging_utils import logging_error, logging_warning
+
 """
-Convert configuration from properties file to an object. 
+Convert configuration from properties file to an object.
 """
 
 
@@ -11,7 +14,7 @@ class AMQAdapter:
     configuration with prefix 'cm.amq-adapter'
     """
 
-    PREFIX: str = 'cm.amq-adapter.'
+    PREFIX: str = "cm.amq-adapter."
 
     def __init__(self, config: configparser.ConfigParser):
         """
@@ -21,10 +24,27 @@ class AMQAdapter:
 
         :param config: config parser to read configuration values
         """
-        self.generator_id = config.getint('CleverMicro-AMQ', self.PREFIX + 'generator-id', fallback=164)
-        self.service_name = config.get('CleverMicro-AMQ', self.PREFIX + 'service-name',
-                                       fallback='amq-python-adapter-' + str(self.generator_id))
-        self.route_mapping = config.get('CleverMicro-AMQ', self.PREFIX + 'route-mapping', fallback=None)
+        self.generator_id = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "generator-id", fallback=164
+        )
+        self.service_name = config.get(
+            "CleverMicro-AMQ",
+            self.PREFIX + "service-name",
+            fallback="amq-python-adapter-" + str(self.generator_id),
+        )
+        self.route_mapping = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "route-mapping", fallback=None
+        )
+        self.require_authenticated_user = config.getboolean(
+            "CleverMicro-AMQ",
+            self.PREFIX + "require-authenticated-user",
+            fallback=False,
+        )
+        # these 2 shall be provided by Docker swarm, format is a random alphanumeric string
+        self.swarm_service_id = os.getenv(
+            "SWARM_SERVICE_ID", default="clever-amqp-service"
+        )
+        self.swarm_task_id = os.getenv("SWARM_TASK_ID", default="python-adapter")
 
     def adapter_prefix(self) -> str:
         return f"{self.service_name}-{self.generator_id}-"
@@ -35,12 +55,11 @@ class Dispatch:
     configuration with prefix 'cm.dispatch'
     """
 
-    PREFIX: str = 'cm.dispatch.'
+    PREFIX: str = "cm.dispatch."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.dispatch.use-dlq=true
-        cm.dispatch.use-confirms=false
         cm.dispatch.amq-host=rabbitmq
         cm.dispatch.amq-port=5672
         cm.dispatch.amq-port-tls=5671
@@ -49,15 +68,32 @@ class Dispatch:
 
         :param config: config parser to read configuration values
         """
-        self.use_dlq = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-dlq', fallback=False)
-        self.use_confirms = config.getboolean('CleverMicro-AMQ', self.PREFIX + 'use-confirms', fallback=False)
-        self.amq_host = config.get('CleverMicro-AMQ', self.PREFIX + 'amq-host', fallback='rabbitmq')
-        self.amq_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port', fallback=5672)
-        self.amq_port_tls = config.getint('CleverMicro-AMQ', self.PREFIX + 'amq-port-tls', fallback=5671)
-        self.service_host = config.get('CleverMicro-AMQ', self.PREFIX + 'service-host', fallback='localhost')
-        self.service_port = config.getint('CleverMicro-AMQ', self.PREFIX + 'service-port', fallback=8080)
+        self.use_dlq = config.getboolean(
+            "CleverMicro-AMQ", self.PREFIX + "use-dlq", fallback=False
+        )
+        self.amq_host = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "amq-host", fallback="rabbitmq"
+        )
+        self.amq_port = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "amq-port", fallback=5672
+        )
+        self.amq_port_tls = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "amq-port-tls", fallback=5671
+        )
+        self.service_host = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "service-host", fallback="localhost"
+        )
+        self.service_port = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "service-port", fallback=8080
+        )
         self.rabbit_mq_user = os.getenv("RABBIT_MQ_USER", "springuser")
         self.rabbit_mq_password = os.getenv("RABBIT_MQ_PASSWORD", "TheCleverWho")
+        self.rabbit_mq_url = (
+            f"amqp://{self.rabbit_mq_user}:{self.rabbit_mq_password}@{self.amq_host}/"
+        )
+        self.download_dir = config.get(
+            "CleverMicro-AMQ", self.PREFIX + "download-dir", fallback="downloaded_files"
+        )
 
 
 class Backpressure:
@@ -65,33 +101,62 @@ class Backpressure:
     configuration with prefix 'cm.backpressure'
     """
 
-    PREFIX: str = 'cm.backpressure.'
+    PREFIX: str = "cm.backpressure."
 
     def __init__(self, config: configparser.ConfigParser):
         """
         cm.backpressure.threshold=20
         cm.backpressure.time-window=3000
-        cm.backpressure.management-service-url=management-service:8080
 
         :param config: config parser to read configuration values
         """
-        self.management_service_url = config.get('CleverMicro-AMQ', self.PREFIX + 'management-service-url', fallback='management-service:8080')
-        self.threshold = config.getint('CleverMicro-AMQ', self.PREFIX + 'threshold', fallback=20)
-        self.time_window = config.getint('CleverMicro-AMQ', self.PREFIX + 'time-window', fallback=3000)
+        self.threshold_threads = config.getint(
+            "CleverMicro-AMQ",
+            self.PREFIX + "threshold",
+            fallback=int(os.getenv("PARALLEL_WORKERS", 1)),
+        )
+        # time-window in seconds, duration between backpressure reports
+        self.time_window = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "time-window", fallback=10
+        )
+
+        # Define maximum CPU usage before triggering backpressure OVERLOAD alert
+        self.threshold_cpu = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "threshold-cpu", fallback=90
+        )
+        # Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+        self.cpu_overload_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "cpu-overload-duration", fallback=30
+        )
+        # Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+        self.idle_duration = config.getint(
+            "CleverMicro-AMQ", self.PREFIX + "idle-duration", fallback=30
+        )
 
 
 class AMQConfiguration:
     """
     Top level configuration context holder / object
     """
+
     def __init__(self, config_file_name: str = None):
         config = configparser.ConfigParser()
         # Read the application.properties file
         if os.path.exists(config_file_name):
             config.read(config_file_name)
         else:
-            print(f" [E] Configuration file not found: {config_file_name}")
-            print(" [W] Defaulting to preset values, which will likely be wrong, causing errors.")
+            # Get the directory of the caller (the module where this function is called)
+            caller_frame = inspect.stack()[1]  # 0 is current frame, 1 is caller
+            caller_module = inspect.getmodule(caller_frame[0])
+            if caller_module:
+                caller_dir = os.path.dirname(os.path.abspath(caller_module.__file__))
+                config_file_path = os.path.join(caller_dir, config_file_name)
+                config.read(config_file_path)
+            else:
+                logging_error(f"Configuration file not found: {config_file_name}")
+                logging_warning(
+                    "Defaulting to preset values, which will likely be wrong, causing errors."
+                )
 
         # Override values with environment variables
         for section in config.sections():

amqp/config/application.properties

@@ -2,17 +2,41 @@
 # CleverMicro AMQ Adapter settings
 # ====================================================================
 [CleverMicro-AMQ]
-cm.amq-adapter.service-name=amq-adapter
-cm.amq-adapter.is-router=false
-cm.amq-adapter.generator-id=164
-cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:cleverswarm.localhost.#:5:0
+# CleverMicro AMQ Adapter settings that identify this AMQ Adapter instance to other adapters
 
+# A name to identify this service for the AMQ Adapter
+cm.amq-adapter.service-name=cleverthis-service-name
+# The CleverMicro AMQ Adapter instance ID, used to identify this instance in the CleverMicro AMQ Adapter cluster
+cm.amq-adapter.generator-id=1
+# Route mapping. For route detail description see section 3.1 in Onboarding Guide for Python:
+# https://docs.cleverthis.com/en/architecture/microservices/onboarding/python
+# NOTE: this is a comma-separated list of route mappings (multiple, comma-separated routes are allowed here)
+cm.amq-adapter.route-mapping=
+# Indicate if AMQ Adapter should respond with HTTP 401 Unauthorized if the request does not contain valid JWT token
+cm.amq-adapter.require-authenticated-user=false
+
+# CleverMicro AMQ Adapter dispatcher, settings related to RabbitMQ connection and optional REST forwarding
 cm.dispatch.use-dlq=true
-cm.dispatch.use-confirms=false
 cm.dispatch.amq-host=rabbitmq
 cm.dispatch.amq-port=5672
 cm.dispatch.amq-port-tls=5671
+cm.dispatch.download-dir=/tmp/downloaded_files
+# Not used for CleveSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverthis-service-name.localhost
+#cm.dispatch.service-port=8080
+
+# Backpressure monitor settings
+# Define maximum number of messages of being processed in parallel before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold=5
+# Define maximum CPU usage (%) before triggering backpressure OVERLOAD alert
+cm.backpressure.threshold-cpu-overload=90
+# Define maximum CPU usage (%) before triggering backpressure IDLE alert
+cm.backpressure.threshold-cpu-idle=10
+# Define the time window between the Backpressure reports, in seconds
+cm.backpressure.time-window=10
+# Define the time window length, in seconds, during which, if CPU usage is consistently above the threshold, the system will trigger OVERLOAD alert
+cm.backpressure.cpu-overload-duration=30
+# Define the time window length, in seconds, during which, if no request is made to the Service, the system will trigger IDLE alert
+cm.backpressure.cpu-idle-duration=30
 
-cm.backpressure.threshold=20
-cm.backpressure.time-window=3000
-cm.backpressure.management-service-url=management-service:8080

amqp/config/application.properties.local

@@ -2,17 +2,18 @@
 # CleverMicro AMQ Adapter settings
 # ====================================================================
 [CleverMicro-AMQ]
-cm.amq-adapter.service-name=amq-adapter
-cm.amq-adapter.is-router=false
+cm.amq-adapter.service-name=amq-adapter-python
 cm.amq-adapter.generator-id=211
-cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:cleverswarm.localhost.#:5:0
+cm.amq-adapter.route-mapping=amq-cleverswarm::amq-cleverswarm:localhost.#:5:0
 
 cm.dispatch.use-dlq=true
 cm.dispatch.use-confirms=false
 cm.dispatch.amq-host=localhost
 cm.dispatch.amq-port=5672
 cm.dispatch.amq-port-tls=5671
+# Not used for CleverSwarm nor CleverBRAG or in tight coupling mode.
+# applicable only in loose coupling mode, specify the destination where to forward the REST request
+#cm.dispatch.service-host=cleverswarm.localhost
+#cm.dispatch.service-port=8080
 
 cm.backpressure.threshold=20
-cm.backpressure.time-window=3000
-cm.backpressure.management-service-url=management-service:8080

amqp/healthcheck.py

@@ -1,9 +1,9 @@
 import sys
 import requests
 
-url = 'http://localhost:8080/amq-adapter-healthcheck'
-headers = {'Authorization': 'Bearer my_access_token'}
-params = {'limit': 10, 'offset': 20}
+url = "http://localhost:8080/amq-adapter-healthcheck"
+headers = {"Authorization": "Bearer my_access_token"}
+params = {"limit": 10, "offset": 20}
 response = requests.get(url, headers=headers, params=params)
 if response.status_code == 200:
     sys.exit(0)

amqp/model/model.py

@@ -1,7 +1,8 @@
 import base64
-
-from typing import List, Dict
+import json
 from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, List
 
 from amqp.model.service_message_type import ServiceMessageType
 from amqp.model.snowflake_id import SnowflakeId
@@ -11,10 +12,155 @@ RS = ":"
 
 
 """
-Define the structure of messages used in Clever Microś
+Define the structure of messages used in Clever Micro
 """
 
 
+@dataclass
+class CleverMicroMessage:
+    DEFAULT_CONTENT_TYPE = ["application/octet-stream"]
+
+    """
+    The common structure of a message that transfers the user request to the Service and back. All data related messages
+    conform to this base structure:
+
+    magic - identifies the message type (REST, RPC, DataResponse)
+    id - unique message ID
+    m_field - method name (for REST style calls) or service name (for RPC style calls) or response code for DataResponse
+    d_field - domain name (for REST style calls) or module name (for RPC style calls) or  error code for DataResponse
+    p_field - path (for REST style calls) or method name (for RPC style calls) or error cause for DataResponse
+    headers - headers of the message (e.g. Content-Type, Accept, etc.)
+    trace_info - trace information (e.g. trace ID, span ID, etc. for Jaeger trace monitor)
+    base64body - the body of the message encoded in base64
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        m_field: str,
+        d_field: str,
+        p_field: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        self._magic = magic
+        self._id: SnowflakeId = id
+        self._m_field = m_field
+        self._d_field = d_field
+        self._p_field = p_field
+        self._headers: Dict[str, List[str]] = headers
+        self._trace_info: Dict[str, str] = trace_info
+        self._base64body: bytes = base64body
+
+    def magic(self):
+        return self._magic
+
+    def id(self):
+        return self._id
+
+    def m_field(self) -> str:
+        return self._m_field
+
+    def d_field(self) -> str:
+        return self._d_field
+
+    def p_field(self) -> str:
+        return self._p_field
+
+    def headers(self) -> Dict[str, List[str]]:
+        return self._headers
+
+    def trace_info(self) -> Dict[str, str]:
+        return self._trace_info
+
+    def base64body(self) -> bytes:
+        return self._base64body
+
+    def body(self) -> bytes:
+        return base64.b64decode(self.base64body())
+
+    def content_type(self) -> str:
+        return self.headers().get("Content-Type", self.DEFAULT_CONTENT_TYPE)[0]
+
+
+class DataMessage(CleverMicroMessage):
+    """
+    The class represents a structure of a message that transfers the user request to the Service.
+    It may also be used to make a call/request between services.
+    The response to this call is in format of DataResponse class.
+
+    See DataMessageFactory for code to instantiate, serialize and deserialize the class
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        method: str,
+        domain: str,
+        path: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        super().__init__(
+            magic, id, method, domain, path, headers, trace_info, base64body
+        )
+
+    def method(self) -> str:
+        return self.m_field()
+
+    def domain(self) -> str:
+        return self.d_field()
+
+    def path(self) -> str:
+        return self.p_field()
+
+    def routing_key(self) -> str:
+        return sanitize_as_rabbitmq_name(f"{self.d_field()}.{self.p_field()}")
+
+
+class DataResponse(CleverMicroMessage):
+    """
+    The response message to REST & RPC style calls (note: the request is made with DataMessage)
+
+    See DataResponseFactory for code to instantiate, serialize and deserialize the class
+    """
+
+    def __init__(
+        self,
+        magic: str,
+        id: SnowflakeId,
+        response_code: str,
+        error: str,
+        error_cause: str,
+        headers: Dict[str, List[str]],
+        trace_info: Dict[str, str],
+        base64body: bytes,
+    ):
+        super().__init__(
+            magic,
+            id,
+            response_code,
+            error,
+            error_cause,
+            headers,
+            trace_info,
+            base64body,
+        )
+
+    def response_code(self) -> str:
+        return self.m_field()
+
+    def error(self) -> str:
+        return self.d_field()
+
+    def error_cause(self) -> str:
+        return self.p_field()
+
+
 @dataclass(frozen=True)
 class AMQRoute:
     """
@@ -28,6 +174,7 @@ class AMQRoute:
     :param timeout: Timeout in milliseconds, how long to wait for reply, or not waiting if less then 1
     :param valid_until: optional timestamp (millis from epoch) after which the route is ignored
     """
+
     component_name: str
     exchange: str
     queue: str
@@ -45,9 +192,11 @@ class AMQRoute:
 
     def __eq__(self, other):
         if isinstance(other, AMQRoute):
-            return (self.component_name == other.component_name
-                    and self.queue == other.queue
-                    and self.key == other.key)
+            return (
+                self.component_name == other.component_name
+                and self.queue == other.queue
+                and self.key == other.key
+            )
         return False
 
     @property
@@ -62,54 +211,32 @@ class AMQRoute:
         return self.__str__()
 
 
-@dataclass
-class AMQMessage:
-    """
-    The class represents a structure of a message that transfers the user request to the Service.
-    It may also be used to make a call/request between services.
-    The response to this call is in format of AMQResponse class.
-    Please see AMQMessageFactory for code to instantiate, serialize and deserialize the class
-    """
-    magic: str
-    id: SnowflakeId
-    method: str
-    domain: str
-    path: str
-    headers: dict[str, List[str]]
-    trace_info: dict[str, str]
-    base64_body: bytes
-
-    def routing_key(self) -> str:
-        return sanitize_as_rabbitmq_name(f"{self.domain}.{self.path}")
-
-    def body(self) -> bytes:
-        return base64.b64decode(self.base64_body)
+class MultipartDataMessage(DataMessage):
+    def __init__(self, message: DataMessage, extra_data: dict):
+        super().__init__(
+            message.magic,
+            message.id,
+            message.method,
+            message.domain,
+            message.path,
+            message.headers,
+            message.trace_info,
+            message.base64_body,
+        )
+        self.extra_data = extra_data
 
 
-@dataclass
-class AMQResponse:
-    """
-    The response message to RPC style call (note: the request is made with AMQMessage)
-    Please see AMQResponseFactory for code to instantiate, serialize and deserialize the class
-    """
-    id: str
-    response_code: int
-    content_type: str
-    response: bytes
-    error: str | None
-    error_cause: str | None
-    trace_info: dict[str, str]
-
-    def body(self) -> bytes:
-        return base64.b64decode(self.response)
+# Ensure backward compatibility
+AMQResponse = DataResponse
 
 
 @dataclass(frozen=True)
-class CleverMicroServiceMessage:
+class ServiceMessage:
     """
     The CleverMicro service message that is used to communicate the internal state of the AMQ channel.
-    In v0.1 it supports service discovery / route registration and Backpressure status notification.
+    In v0.2 it supports service discovery / route registration and Backpressure status notification.
     """
+
     id: SnowflakeId = None
     payload_type: int = 0
     message_type: ServiceMessageType = ServiceMessageType.UNKNOWN
@@ -120,3 +247,76 @@ class CleverMicroServiceMessage:
 
     def is_valid(self) -> bool:
         return self.message_type is not None
+
+
+class ScalingRequestAlert(Enum):
+    OVERLOAD = "OVERLOAD"
+    IDLE = "IDLE"
+    UPDATE = "UPDATE"
+    SCALE_UP = "SCALE_UP"
+    SCSLE_DOWN = "SCALE_DOWN"
+
+
+@dataclass(frozen=True)
+class ScalingRequest:
+    service_id: str
+    task_id: str
+    max_availability: int
+    current_availability: int
+    request_type: ScalingRequestAlert
+    version: int = 1
+
+    def to_json(self) -> str:
+        """Converts the object to a JSON string matching the Java structure"""
+        return json.dumps(
+            {
+                "version": self.version,
+                "serviceId": self.service_id,
+                "taskId": self.task_id,
+                "maxAvailability": self.max_availability,
+                "currentAvailability": self.current_availability,
+                "requestType": self.request_type.value,  # Using .value for Enum serialization
+            },
+            indent=2,
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "ScalingRequest":
+        """Creates a ScalingRequest from a JSON string"""
+        data = json.loads(json_str)
+        return cls(
+            service_id=data["serviceId"],
+            task_id=data["taskId"],
+            max_availability=data["maxAvailability"],
+            current_availability=data["currentAvailability"],
+            request_type=ScalingRequestAlert(
+                data["requestType"]
+            ),  # Convert string to Enum
+        )
+
+
+class AMQErrorMessage:
+    """
+    The error response message to RPC style call (note: the request is made with DataMessage)
+    """
+
+    def __init__(
+        self,
+        response_code: int,
+        error: str,
+        detail: str,
+        content_type: str = "application/json",
+    ):
+        self.response_code = response_code
+        self.error = error
+        self.detail = detail
+        self.content_type = content_type
+
+    def serialize(self) -> bytes:
+        return json.dumps(
+            {
+                "response_code": self.response_code,
+                "error": self.error,
+                "detail": self.detail,
+            }
+        ).encode("utf-8")

amqp/model/snowflake_id.py

@@ -5,9 +5,12 @@ class SnowflakeId:
     """
     see https://en.wikipedia.org/wiki/Snowflake_ID
     """
-    SNOWFLAKE_ID_WIDTH = 10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
 
-    def __init__(self, hi_bits_or_bytes = None, lo_bits = None):
+    SNOWFLAKE_ID_WIDTH = (
+        10  # with in bytes (8 bits each), can be more than long int width (8 bytes)
+    )
+
+    def __init__(self, hi_bits_or_bytes=None, lo_bits=None):
         self.bits = [0, 0]
         if lo_bits is None:
             if hi_bits_or_bytes is not None:
@@ -39,13 +42,17 @@ class SnowflakeId:
         for i in range(0, expected_length, 2):
             if i == expected_length - (2 * 8):
                 n = 0
-            bits[n] = (bits[n] << 8) | int(_input[i:i + 2], 16)
+            bits[n] = (bits[n] << 8) | int(_input[i : i + 2], 16)
 
         return SnowflakeId(bits[1], bits[0])
 
     def __str__(self):
         _hex = f"{self.bits[1]:016X}{self.bits[0]:016X}"
-        _val = _hex if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH):]
+        _val = (
+            _hex
+            if len(_hex) <= 2 * SnowflakeId.SNOWFLAKE_ID_WIDTH
+            else _hex[(-2 * SnowflakeId.SNOWFLAKE_ID_WIDTH) :]
+        )
         return _val.lower()
 
     def __eq__(self, other):

amqp/model/uploaded_file.py

@@ -1,21 +0,0 @@
-import io
-
-
-class UploadedFile:
-    def __init__(self, filename: str = None):
-        self.filename = filename
-        self.body_b64 = io.BytesIO()
-
-
-def from_bytes(input_bytes: bytes) -> UploadedFile:
-    """
-    Builds the UploadedFile from its serialized (byte array) form.
-    :param input_bytes: serialized message
-    :return: UploadedFile instance
-    """
-    prolog_len = (input_bytes[0] << 8) + input_bytes[1]
-    metadata = input_bytes[2:prolog_len + 2].decode()
-    body_b64 = input_bytes[prolog_len + 2:]
-
-    return UploadedFile()
-

amqp/rabbitmq/rabbit_mq_client.py

@@ -0,0 +1,238 @@
+import logging
+
+import aio_pika
+from aio_pika.abc import AbstractRobustQueue, AbstractRobustExchange
+from pika.exchange_type import ExchangeType
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.logging_utils import (
+    logging_error,
+    logging_debug,
+    logging_info,
+    logging_warning,
+)
+from amqp.model.model import DataMessage, AMQRoute
+from amqp.router.router_base import (
+    AMQ_REPLY_TO_EXCHANGE,
+    DLQ_EXCHANGE,
+    AMQ_RPC_EXCHANGE,
+)
+from amqp.router.router_consumer import RouterConsumer
+from amqp.router.utils import sanitize_as_rabbitmq_name
+
+
+class RabbitMQClient:
+    PREFETCH_COUNT = 1
+    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
+
+    def __init__(self, configuration):
+        self.channel: aio_pika.RobustChannel | None = None
+        self.connection: aio_pika.RobustConnection | None = None
+        self.rpc_exchange: aio_pika.RobustExchange | None = None
+        self.reply_exchange: aio_pika.RobustExchange | None = None
+        self.amq_configuration = configuration
+        logging.info("*******************************************")
+        logging.info(
+            "******** RabbitMQClient.init(): %s",
+            self.amq_configuration.dispatch.amq_host,
+        )
+        logging.info("*******************************************")
+        self.router = RouterConsumer(configuration)
+        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+
+        self.reply_to_exchange: AbstractRobustExchange | None = (
+            None  # when sending response to RPC request
+        )
+        self.reply_to_queue: AbstractRobustQueue | None = (
+            None  # when receiving response to RPC request
+        )
+
+    async def init(self, loop) -> AbstractRobustExchange:
+        await self.setup_amq_channel(loop)
+        await self.router.init_routing_paths_consumer(self.router.channel)
+        cm_reply_to_queue = self.router.get_reply_to_queue_name()
+        self.reply_to_queue = await self.channel.declare_queue(
+            name=cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False
+        )
+        self.reply_to_exchange = await self.channel.declare_exchange(
+            AMQ_REPLY_TO_EXCHANGE, type=ExchangeType.direct
+        )
+        await self.reply_to_queue.bind(
+            exchange=self.reply_to_exchange, routing_key=cm_reply_to_queue
+        )
+        return self.reply_to_exchange
+
+    def cleanup(self):
+        logging_info("RabbitMQClient.cleanup()")
+        self.router.cleanup()  # de-register route(s)
+        if self.router.is_open(self.channel):
+            try:
+                self.channel.close()
+                if not self.connection.is_closed:
+                    self.connection.close()
+            except Exception as e:
+                logging_error(
+                    "RabbitMQClient.PreDestroy cleanup - connection close error: %s",
+                    e,
+                )
+
+    async def register_inbound_routes(self, route_mapping: str, callback):
+        """
+        Register inbound routes for the AMQ adapter. The route mapping is a comma-separated string that contains
+        list of routes. A route is the mapping of domain and path to the exchange and queue name.
+        This method declare (and thus create) the queue and exchange, bind them together using the
+        regex-like routing expression given in the route.
+        After this the Service is listening on (consuming from) the queue for incoming messages.
+        """
+        if self.router.is_open(self.channel):
+            queue_args = (
+                self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
+            )
+            logging_info(
+                "RabbitMQClient register routes, cfg mapping: [%s]", route_mapping
+            )
+            # convert the comma-separated string into list of AMQRoute objects
+            requested_routes = self.router.unwrap_route_list(route_mapping)
+            for route in requested_routes:
+                try:
+                    # Note: AMQRoute allows one of the exchange or queue to be None, in which case the default is used
+                    queue_name = (
+                        route.queue
+                        if route.queue
+                        else self.router.get_consume_queue_name()
+                    )
+                    logging_info("RabbitMQClient register route: [%s]", route)
+                    queue: AbstractRobustQueue = await self.channel.declare_queue(
+                        name=queue_name,
+                        durable=True,
+                        exclusive=False,
+                        auto_delete=False,
+                        arguments=queue_args,
+                    )
+                    _exchange_name: str = (
+                        route.exchange if route.exchange else AMQ_RPC_EXCHANGE
+                    )
+                    exchange: AbstractRobustExchange = (
+                        await self.channel.declare_exchange(
+                            name=_exchange_name, type=ExchangeType.topic
+                        )
+                    )
+                    await queue.bind(exchange, route.key)
+                    _c_tag = await queue.consume(callback, no_ack=False)
+                    # Register the route with the router and publish its detail to other adapters
+                    await self.router.register_route(
+                        AMQRoute(
+                            route.component_name,
+                            _exchange_name,
+                            queue_name,
+                            sanitize_as_rabbitmq_name(route.key),
+                            route.timeout,
+                            route.valid_until,
+                        )
+                    )
+                    logging_info(
+                        " [%s] AMQ connection initialized for route: %s, listens on: %s/%s",
+                        _c_tag,
+                        route,
+                        exchange,
+                        queue_name,
+                    )
+                except Exception as err:
+                    logging_error("Callback registration INCOMPLETE, %s", err)
+        else:
+            logging_warning("Channel is null, cannot register routes.")
+
+    async def register_data_reply_callback(self, rpc_callback):
+        _c_tag = await self.reply_to_queue.consume(callback=rpc_callback, no_ack=False)
+        logging_info(
+            " [%s] RabbitMQClient register data reply callback on queue: %s",
+            _c_tag,
+            self.reply_to_queue.name,
+        )
+
+    async def setup_amq_channel(self, loop):
+        """
+        Set up the RabbitMQ connection
+        :return: nothing, it applies changes to instance variables
+        """
+        self.connection = await aio_pika.connect_robust(
+            host=self.amq_configuration.dispatch.amq_host,
+            port=self.amq_configuration.dispatch.amq_port,
+            login=self.amq_configuration.dispatch.rabbit_mq_user,
+            password=self.amq_configuration.dispatch.rabbit_mq_password,
+            loop=loop,
+        )
+        logging_debug(
+            "RabbitMQClient.setupAMQ, got connection, open=%s",
+            self.connection.is_closed,
+        )
+        self.channel = await self.connection.channel()
+        await self.channel.set_qos(prefetch_count=self.PREFETCH_COUNT)
+        await self.router.init_internal_routing_paths(self.channel)
+        self.router.set_route_added_notifier(
+            lambda route: logging_debug(
+                " RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
+                route.exchange,
+            )
+        )
+
+    async def send_message(self, message: DataMessage):
+        """
+        The API routine that sends a datagram message (DataMessage) to the destination. The Destination
+        is determined from the DataMessage metadata section, using `domain` and `path` as key.
+        :param message: message to send
+        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
+         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
+        """
+        try:
+            route = self.router.find_route_by_message(message)
+            if route and self.rpc_exchange:
+                if route.timeout <= 0:
+                    # No response expected
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(
+                        f"Msg Publish (no-reply), messageId: {message.id}, to: {route}"
+                    )
+                else:
+                    # This is RPC call, there should be response
+                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
+                        body=DataMessageFactory.serialize(message),
+                        content_encoding="utf-8",
+                        delivery_mode=2,
+                        correlation_id=str(message.id),
+                        reply_to=self.router.get_reply_to_queue_name(),
+                    )
+                    await self.rpc_exchange.publish(
+                        message=pika_message,
+                        routing_key=self.router.get_routing_key(
+                            message
+                        ),  # context path is a routing key
+                    )
+                    logging_info(
+                        "Msg Publish (RPC call), messageId: %s, to: %s",
+                        message.id.as_string(),
+                        route.as_string(),
+                    )
+            else:
+                logging_warning(
+                    "Message not sent, no route for %s/%s", message.domain, message.path
+                )
+
+        except Exception as err:
+            logging_error("Send message failed: %s", err)
+
+    async def request_routes(self):
+        await self.router.request_routes_from_adapters()
+
+    def find_route(self, domain: str, path: str) -> AMQRoute:
+        return self.router.find_route(domain, path)

amqp/rabbitmq/rabbit_mq_consumer.py

@@ -1,80 +0,0 @@
-import logging
-import pika
-
-from aio_pika.abc import AbstractRobustQueue, AbstractRobustExchange
-from pika.exchange_type import ExchangeType
-from amqp.model.model import AMQRoute
-from amqp.rabbitmq.rabbit_mq_producer import RabbitMQProducer
-from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE, DLQ_EXCHANGE, AMQ_RPC_EXCHANGE
-from amqp.router.router_consumer import RouterConsumer
-from amqp.router.utils import sanitize_as_rabbitmq_name
-
-
-class RabbitMQConsumer(RabbitMQProducer):
-    PREFETCH_COUNT = 1
-    DLQ_ARGS = {"x-dead-letter-exchange": DLQ_EXCHANGE}
-
-    def __init__(self, configuration):
-        super().__init__(configuration, RouterConsumer(configuration))
-        self.reply_to_exchange: AbstractRobustExchange | None = None # when sending response to RPC request
-        self.reply_to_queue: AbstractRobustQueue | None= None # when receiving response to RPC request
-
-    async def init(self, loop) -> AbstractRobustExchange:
-        await self.setup_amq_channel(loop)
-        cm_reply_to_queue = self.router.get_reply_to_queue_name()
-        self.reply_to_queue = await self.channel.declare_queue(
-            name=cm_reply_to_queue, durable=True, exclusive=False, auto_delete=False
-        )
-        self.reply_to_exchange = await self.channel.declare_exchange(
-            AMQ_REPLY_TO_EXCHANGE, type=pika.exchange_type.ExchangeType.direct
-        )
-        await self.reply_to_queue.bind(exchange=self.reply_to_exchange, routing_key=cm_reply_to_queue)
-        return self.reply_to_exchange
-
-    def cleanup(self):
-        logging.info(" [*] RabbitMQConsumer.cleanup()")
-        self.router.cleanup()  # de-register route(s)
-        if self.router.is_open(self.channel):
-            try:
-                self.channel.close()
-            except Exception as e:
-                logging.error(" [E] RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    async def register_inbound_routes(self, route_mapping: str, callback):
-        if self.router.is_open(self.channel):
-            queue_args = self.DLQ_ARGS if self.amq_configuration.dispatch.use_dlq else None
-            logging.info(" [*] RabbitMQConsumer register routes, cfg mapping: [%s]", route_mapping)
-            requested_routes = self.router.unwrap_route_list(route_mapping)
-            for route in requested_routes:
-                try:
-                    queue_name = route.queue if route.queue else self.router.get_consume_queue_name()
-                    logging.info(" [*] RabbitMQConsumer register route: [%s]", route)
-                    queue: AbstractRobustQueue = await self.channel.declare_queue(name=queue_name,
-                                                                                  durable=True,
-                                                                                  exclusive=False,
-                                                                                  auto_delete=False,
-                                                                                  arguments=queue_args)
-                    exchange_name: str = route.exchange if route.exchange else AMQ_RPC_EXCHANGE
-                    exchange: AbstractRobustExchange = await self.channel.declare_exchange(
-                        name=exchange_name, type=ExchangeType.topic
-                    )
-                    await queue.bind(exchange, route.key)
-                    await queue.consume(callback, no_ack=False)
-                    await self.router.register_route(
-                        AMQRoute(
-                            route.component_name, exchange_name, queue_name,
-                            sanitize_as_rabbitmq_name(route.key),
-                            route.timeout, route.valid_until
-                        )
-                    )
-                    logging.info(" [*] AMQ connection initialized for route: %s, listens on: %s/%s",
-                                 route, exchange, queue_name)
-                except Exception as err:
-                    logging.error(" [E] Callback registration INCOMPLETE, %s", err)
-        else:
-            logging.warning(" [W] Channel is null, cannot register routes.")
-
-    async def register_rpc_handler(self, rpc_consumer):
-        await self.reply_to_queue.consume(callback=rpc_consumer,
-            no_ack=False
-        )

amqp/rabbitmq/rabbit_mq_producer.py

@@ -1,104 +0,0 @@
-import logging
-import aio_pika
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage, AMQRoute
-from amqp.router.router_producer import RouterProducer
-
-
-class RabbitMQProducer:
-    PREFETCH_COUNT = 1
-
-    def __init__(self, amq_configuration: AMQConfiguration, router: RouterProducer = None):
-        self.channel: aio_pika.RobustChannel | None = None
-        self.connection: aio_pika.RobustConnection | None = None
-        self.rpc_exchange: aio_pika.RobustExchange | None = None
-        self.reply_exchange: aio_pika.RobustExchange | None = None
-        self.amq_configuration = amq_configuration
-        logging.info("*******************************************")
-        logging.info("******** RabbitMQProducer.init(): %s",
-                     self.amq_configuration.dispatch.amq_host)
-        logging.info("*******************************************")
-        self.router = router or RouterProducer(amq_configuration)
-        # this.meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
-
-    async def init(self, loop):
-        await self.setup_amq_channel(loop)
-
-    def cleanup(self):
-        if self.connection:
-            try:
-                self.connection.close()
-            except Exception as e:
-                logging.error("RabbitMQConsumer.PreDestroy cleanup - connection close error: %s", e)
-
-    async def setup_amq_channel(self, loop):
-        """
-        Set up the RabbitMQ connection
-        :return: nothing, it applies changes to instance variables
-        """
-        self.connection = await aio_pika.connect_robust(
-            host=self.amq_configuration.dispatch.amq_host,
-            port=self.amq_configuration.dispatch.amq_port,
-            login=self.amq_configuration.dispatch.rabbit_mq_user,
-            password=self.amq_configuration.dispatch.rabbit_mq_password,
-            loop=loop
-        )
-        logging.info("RabbitMQProducer.setupAMQ, got connection, open=%s", self.connection.is_closed)
-        self.channel = await self.connection.channel()
-        await self.channel.set_qos(prefetch_count=self.PREFETCH_COUNT)
-        await self.router.init_routing_paths(self.channel)
-        self.router.set_route_added_notifier(
-            lambda route: logging.info(" [DBG] ---------- RouteAdded, setting up metrics: amqp.adapter.exchange.%s",
-                                       route.exchange))
-
-    async def send_message(self, message: AMQMessage):
-        """
-        The API routine that sends a datagram message (AMQMessage) to the destination. The Destination
-        is determined from the AMQMessage metadata section, using `domain` and `path` as key.
-        :param message: message to send
-        :return: nothing (void) as submitting message to RabbitMQ exchange does not return anything
-         meaningful. There will be asynch ACK later that will confirm the message was sent to RMQ.
-        """
-        try:
-            route = self.router.find_route_by_message(message)
-            if route and self.rpc_exchange:
-                if route.timeout <= 0:
-                    # No response expected
-                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
-                        body=AMQMessageFactory.serialize(message),
-                        content_encoding='utf-8',
-                        delivery_mode=2
-                    )
-                    await self.rpc_exchange.publish(
-                        message=pika_message,
-                        routing_key=self.router.get_routing_key(message),  # context path is a routing key
-                    )
-                    logging.info(f" [x] basicPublish (no-reply), messageId: {message.id}, to: {route}")
-                else:
-                    # This is RPC call, there should be response
-                    pika_message: aio_pika.message.Message = aio_pika.message.Message(
-                        body=AMQMessageFactory.serialize(message),
-                        content_encoding='utf-8',
-                        delivery_mode=2,
-                        correlation_id=str(message.id),
-                        reply_to=self.router.get_reply_to_queue_name()
-                    )
-                    await self.rpc_exchange.publish(
-                        message=pika_message,
-                        routing_key=self.router.get_routing_key(message),  # context path is a routing key
-                    )
-                    logging.info(" [x] basicPublish (RPC call), messageId: %s, to: %s",
-                                 message.id.as_string(), route.as_string())
-            else:
-                logging.warning("Message not sent, no route for %s/%s", message.domain, message.path)
-
-        except Exception as err:
-            logging.error("Send message failed: %s", err)
-
-    async def request_routes(self):
-        await self.router.request_routes_from_adapters()
-
-    def find_route(self, domain: str, path: str) -> AMQRoute:
-        return self.router.find_route(domain, path)

amqp/router/route_database.py

@@ -1,7 +1,7 @@
-import logging
 import re
 from typing import Set
 
+from amqp.adapter.logging_utils import logging_info
 from amqp.model.model import AMQRoute
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -15,7 +15,7 @@ class RouteDatabase:
 
     def pattern(self, routing_key: str) -> re.Pattern:
         breaker = False
-        tokens = routing_key.split('.')
+        tokens = routing_key.split(".")
         counter = len(tokens)
         regex_parts = []
 
@@ -40,16 +40,20 @@ class RouteDatabase:
             return re.compile(".*")
         return re.compile(regex)
 
-    def matches(self, routing_key_from_route: str, routing_key_from_message: str) -> bool:
+    def matches(
+        self, routing_key_from_route: str, routing_key_from_message: str
+    ) -> bool:
         route_pattern = self.pattern(routing_key_from_route)
-        return bool(route_pattern.match(sanitize_as_rabbitmq_name(routing_key_from_message)))
+        return bool(
+            route_pattern.match(sanitize_as_rabbitmq_name(routing_key_from_message))
+        )
 
     def wrap_routes(self) -> str:
         return ",".join(str(route) for route in self.routes)
 
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         routing_key = sanitize_as_rabbitmq_name(f"{domain}.{path}")
-        logging.info(f" [DBG] findRoute key:{routing_key} in:{self.wrap_routes()}")
+        logging_info(f"findRoute key:{routing_key} in:{self.wrap_routes()}")
         for route in self.routes:
             if self.matches(route.key, routing_key):
                 return route

amqp/router/router_base.py

@@ -1,8 +1,8 @@
-import logging
 from typing import Callable, List, Set
 
-from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
-from amqp.model.model import AMQRoute, AMQMessage
+from amqp.adapter.amq_route_factory import NULL_ROUTE, AMQRouteFactory
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.model.model import AMQRoute, DataMessage
 from amqp.router.route_database import RouteDatabase
 from amqp.router.utils import sanitize_as_rabbitmq_name
 
@@ -26,7 +26,7 @@ class RouterBase:
         self.route_added_notifier: Callable[[AMQRoute], None] | None = None
         self.route_removed_notifier: Callable[[AMQRoute], None] | None = None
 
-    def get_routing_key(self, message: AMQMessage) -> str:
+    def get_routing_key(self, message: DataMessage) -> str:
         return sanitize_as_rabbitmq_name(f"{message.domain}.{message.path}")
 
     def wrap_routes(self) -> str:
@@ -36,8 +36,11 @@ class RouterBase:
         return ",".join(str(route) for route in self.own_routes)
 
     def unwrap_route_list(self, route_string: str) -> List[AMQRoute]:
-        logging.info(" [DBG] RouterMaster.unwrapRoutes, route(s): %s", route_string)
-        routes = [AMQRouteFactory.from_string(route_str) for route_str in route_string.split(",")]
+        logging_debug("RouterMaster.unwrapRoutes, route(s): %s", route_string)
+        routes = [
+            AMQRouteFactory.from_string(route_str)
+            for route_str in route_string.split(",")
+        ]
         return [route for route in routes if route != NULL_ROUTE]
 
     def get_routes(self) -> Set[AMQRoute]:
@@ -46,37 +49,48 @@ class RouterBase:
     def find_route(self, domain: str, path: str) -> AMQRoute | None:
         return self.route_database.find_route(domain, path)
 
-    def find_route_by_message(self, message: AMQMessage) -> AMQRoute | None:
+    def find_route_by_message(self, message: DataMessage) -> AMQRoute | None:
         return self.route_database.find_route(message.domain, message.path)
 
     def add_routes(self, message: str) -> None:
         try:
             amq_route_list = self.unwrap_route_list(message)
-            logging.info(" [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message)
+            logging_info(
+                " [DBG] RouterMaster.addRoute [%d]: %s", len(amq_route_list), message
+            )
             for route in amq_route_list:
-                logging.info(" [*] RouterMaster.addRoute, adding route: %s", route)
+                logging_info("RouterMaster.addRoute, adding route: %s", route)
                 if not route.exchange and not route.queue and not route.key:
-                    logging.error("Cannot setup AMQ Exchange for '%s', at least one of "
-                                  "Exchange, Queue and/or Key must be present in the routing string. "
-                                  "Expected format is '%s'", route, AMQRoute.FORMAT)
+                    logging_error(
+                        "Cannot setup AMQ Exchange for '%s', at least one of "
+                        "Exchange, Queue and/or Key must be present in the routing string. "
+                        "Expected format is '%s'",
+                        route,
+                        AMQRoute.FORMAT,
+                    )
                 else:
                     if route.exchange != DLQ_EXCHANGE:
                         is_new_route = self.route_database.add_route(route)
                         if is_new_route and self.route_added_notifier:
                             self.route_added_notifier(route)
         except Exception as err:
-            logging.error(" [E] add_routes error: %s", err)
+            logging_error("add_routes error: %s", err)
 
     def set_route_added_notifier(self, callback: Callable[[AMQRoute], None]):
         self.route_added_notifier = callback
 
     def remove_routes(self, message: str) -> None:
         try:
-            to_remove = [AMQRouteFactory.from_string(route_str) for route_str in message.split(",")]
+            to_remove = [
+                AMQRouteFactory.from_string(route_str)
+                for route_str in message.split(",")
+            ]
             removed_count = sum(1 for route in to_remove if self.remove_route(route))
-            logging.info(f" [*] RouterMaster.removeRoutes(): {message}, removed={removed_count}")
+            logging_info(
+                f"RouterMaster.removeRoutes(): {message}, removed={removed_count}"
+            )
         except IOError as err:
-            logging.info(f" [E] Can't remove route(s): {err}")
+            logging_error(f"Can't remove route(s): {err}")
 
     def remove_route(self, route: AMQRoute) -> bool:
         is_route_removed = self.route_database.remove_route(route)

amqp/router/router_consumer.py

@@ -1,136 +1,216 @@
-import logging
 from typing import Set
 
-from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractIncomingMessage
+from aio_pika.abc import (
+    AbstractIncomingMessage,
+    AbstractRobustChannel,
+    AbstractRobustQueue,
+)
+
 from amqp.adapter.amq_route_factory import AMQRouteFactory
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage, AMQRoute
+from amqp.model.model import AMQRoute, ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
-from amqp.router.router_base import CM_C_REQ_QUEUE, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, ANY_RECIPIENT
+from amqp.router.router_base import (
+    ANY_RECIPIENT,
+    CM_C_REQ_QUEUE,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+)
 from amqp.router.router_producer import RouterProducer
 
 
 class RouterConsumer(RouterProducer):
     """
-     * Contains Consumer logic. The logic here:
-     * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
-     * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
-     * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
-     * notify the other (excluding self) Producers about its presence.
-     *
-     * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
-     * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
-     * capability is also required.
+    * Contains Consumer logic. The logic here:
+    * Sets a listener on CM_C_REQ_QUEUE queue to receive routing data requests
+    * Upon receipt of routing data request, responds on CM_RESPONSE_EXCHANGE with local routing data.
+    * It also proactively pushes the local routing data to CM_RESPONSE_EXCHANGE on startup to
+    * notify the other (excluding self) Producers about its presence.
+    *
+    * Consumer is also Producer (Consumer sits alongside the CM Service, and anytime the Service needs
+    * to communicate with other Service, it can do so via the Adapter). Therefore, the Producer
+    * capability is also required.
     """
 
     def __init__(self, configuration: AMQConfiguration):
         super().__init__(configuration)
         self.amqRouteFactory = AMQRouteFactory()
-        logging.info(" [*] RouterConsumer.<init>")
+        logging_info("RouterConsumer.<init>")
 
     async def init_routing_paths_consumer(self, _channel: AbstractRobustChannel):
         """
-         * Initialize router according to Consumer mode.
+        * Initialize router according to Consumer mode.
         """
-
-        await self.init_routing_paths_consumer(_channel)
-        logging.info(" [*] RouterConsumer.initConsumerRouter, channel open: %s", self.is_open(_channel))
+        logging_info(
+            "RouterConsumer.initConsumerRouter, channel open: %s",
+            self.is_open(_channel),
+        )
         if self.is_open(_channel):
             try:
-                #  1. declare a Queue that gives the RoutingDAta requests
-                cm_request_queue: str = self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
-                queue: AbstractRobustQueue = await self.channel.declare_queue(name=cm_request_queue, durable=True, exclusive=False, auto_delete=False)
-                # 1a. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                await queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
+                #  1. declare a Queue to receive the RoutingData requests
+                _cm_request_queue: str = (
+                    self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_REQ_QUEUE
+                )
+                _queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_request_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                )
+                # 1a. bind queue to CM_REQUEST_EXCHANGE w/o any conditions (empty routing key)
+                await _queue.bind(exchange=CM_REQUEST_EXCHANGE, routing_key="")
                 # 1b. Service adapter mode - listens on Request queue and replies to Response queue
-                logging.info(" [*] RouterConsumer listens for RouteDataRequests on: %s", cm_request_queue)
-                await queue.consume(callback=self.handle_routing_data_request_message, no_ack=False)
+                _c_tag = await _queue.consume(
+                    callback=self.handle_routing_data_request_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] RouterConsumer listens for RouteDataRequests on: %s",
+                    _c_tag,
+                    _cm_request_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterConsumer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterConsumer FATAL: cannot initialize, %s", ioe)
         else:
-            logging.error(" [E] RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error(
+                "RouterConsumer FATAL: cannot initialize, RabbitMQ channel is not open."
+            )
 
     """
      * ServiceMessageType.ROUTING_DATA_REQUEST  handler  listening on queue 'CleverMicroRequest'
     """
 
-    async def handle_routing_data_request_message(self, message: AbstractIncomingMessage):
+    async def handle_routing_data_request_message(
+        self, message: AbstractIncomingMessage
+    ):
         # some debug statements, TODO - remove for prod release
         delivery_tag = message.delivery_tag
         debug = "Delivery.Properties = " + message.properties.__str__()
-        logging.info(" [DBG] ******[%] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
-                     delivery_tag, message.consumer_tag, message.properties, debug)
+        logging_debug(
+            " ******[%s] (ROUTING_DATA_REQ) tag: %s, env: %s, props: %s",
+            delivery_tag,
+            message.consumer_tag,
+            message.properties,
+            debug,
+        )
 
         await message.ack(False)
 
-        cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(
+            message.body
+        )
         route_mapping: str = self.wrap_own_routes()
-        logging.info(" [DBG] ******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
-                     delivery_tag, self.service_message_factory.to_string(cm_message), route_mapping)
+        logging_debug(
+            "******[%s] (ROUTING_DATA_REQ) msg=%s, routeMap=[%s]",
+            delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            route_mapping,
+        )
         # Reply only if self adapter has own routeMapping (e.g. API-gateway-side adapter has none)
-        if cm_message.is_valid() and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST and route_mapping:
-            try:
-                service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                    ServiceMessageType.ROUTING_DATA, route_mapping, cm_message.reply_to
+        if (
+            cm_message.is_valid()
+            and cm_message.message_type == ServiceMessageType.ROUTING_DATA_REQUEST
+            and route_mapping
+        ):
+            if cm_message.reply_to != self.amq_configuration.amq_adapter.service_name:
+                try:
+                    service_message: ServiceMessage = self.service_message_factory.of(
+                        ServiceMessageType.ROUTING_DATA,
+                        route_mapping,
+                        cm_message.reply_to,
+                    )
+                    await self.publish_service_message(
+                        service_message, self.cm_response_exchange
+                    )
+                except Exception as err:
+                    logging_error(
+                        "******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
+                        delivery_tag,
+                        CM_RESPONSE_EXCHANGE,
+                        err,
+                    )
+            else:
+                logging_info(
+                    "******[%s] (ROUTING_DATA_REQ) [%s] Ignore request originated by ourselves.",
+                    delivery_tag,
+                    str(message.body),
                 )
-                await self.publish_service_message(service_message, self.cm_response_exchange)
-            except Exception as err:
-                logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) Response to queue: %s failed: %s",
-                              delivery_tag, CM_RESPONSE_EXCHANGE, err)
-
         else:
-            logging.error(" [E] ******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
-                          delivery_tag, str(message.body))
+            logging_error(
+                "******[%s] (ROUTING_DATA_REQ) [%s] produces invalid or no CM msg type.",
+                delivery_tag,
+                str(message.body),
+            )
 
     async def register_route(self, route: AMQRoute):
         """
-         * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
-         * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
-         * :param route: route to register with Master
+        * Called in Consumer (Service Adapter) context. Registers the routing data for self Service
+        * with the Master router by sending own routing data to Master via CleverMicroResponse queue.
+        * :param route: route to register with Master
         """
         try:
             if self.is_open(self.channel) and route.queue:
-                serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+                serviceMessage: ServiceMessage = self.service_message_factory.of(
                     ServiceMessageType.ROUTING_DATA, route.as_string(), ANY_RECIPIENT
                 )
-                await self.publish_service_message(serviceMessage, self.cm_response_exchange)
+                await self.publish_service_message(
+                    serviceMessage, self.cm_response_exchange
+                )
                 self.add_own_route(route)
-                logging.info(" [DBG] RouterConsumer registered Route: %s", route)
+                logging_info("RouterConsumer registered Route: %s", route)
             else:
-                logging.warning(" [W] RouterConsumer couldn't register route data: %s channel isn't open.",
-                                CM_RESPONSE_EXCHANGE)
+                logging_warning(
+                    "RouterConsumer couldn't register route data: %s channel isn't open.",
+                    CM_RESPONSE_EXCHANGE,
+                )
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed register route data: %s", ioe)
+            logging_error("RouterConsumer failed register route data: %s", ioe)
 
     async def remove_route_on_cleanup(self, route: AMQRoute) -> bool:
         """
-         * Unregister / remove route from the list
-         * :param route: route to remove
-         * :return result of remove operation
+        * Unregister / remove route from the list
+        * :param route: route to remove
+        * :return result of remove operation
         """
-        logging.info(" [DBG] RouterConsumer.removeRoute, route: %s", route.as_string())
+        logging_info("RouterConsumer.removeRoute, route: %s", route.as_string())
         try:
-            service_message: CleverMicroServiceMessage = self.service_message_factory.of(
-                ServiceMessageType.ROUTING_DATA_REMOVE,
-                route.as_string(),
-                ANY_RECIPIENT
+            service_message: ServiceMessage = self.service_message_factory.of(
+                ServiceMessageType.ROUTING_DATA_REMOVE, route.as_string(), ANY_RECIPIENT
             )
-            if not await self.publish_service_message(service_message, self.cm_response_exchange):
-                logging.warning(" [W] RouterConsumer couldn't remove current route: %s, channel not open.",
-                                route)
+            if not await self.publish_service_message(
+                service_message, self.cm_response_exchange
+            ):
+                logging_warning(
+                    "RouterConsumer couldn't remove current route: %s, channel not open.",
+                    route,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterConsumer failed remove current route: %s", ioe)
+            logging_error("RouterConsumer failed remove current route: %s", ioe)
 
         result: bool = self.remove_route(route)
         return result
 
     def cleanup(self):
         """
-         * Cleanup - de-register routes with master.
+        * Cleanup - de-register routes with master.
         """
         routes: Set[AMQRoute] = self.get_routes()
-        logging.info(" [DBG] RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes())
-        not_removed = sum(1 for r in (self.remove_route_on_cleanup(route) for route in routes) if not r)
+        logging_info(
+            "RouterConsumer.cleanup() [%d]: %s", len(routes), self.wrap_routes()
+        )
+        not_removed = sum(
+            1
+            for r in (self.remove_route_on_cleanup(route) for route in routes)
+            if not r
+        )
         if not_removed > 0:
-            logging.info(" [W] %d route(s) not removed. self may lead to failed delivery on self route", not_removed)
+            logging_warning(
+                "%d route(s) not removed. self may lead to failed delivery on self route",
+                not_removed,
+            )

amqp/router/router_producer.py

@@ -1,56 +1,80 @@
 """
- * Maps the request to RabbitMQ exchange. Handles the service discovery logic.
- *
- * Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
- * facing side that receives the requests and routes it to queue associated with requested service.
- * 'Service' mode is consumer side for given service, which receives the message from AMQ and
- * forwards it to the application.
- """
-import logging
+* Maps the request to RabbitMQ exchange. Handles the service discovery logic.
+*
+* Router can be either in 'Master' or 'Service (Consumer)' modes. 'Master' mode is the outward
+* facing side that receives the requests and routes it to queue associated with requested service.
+* 'Service' mode is consumer side for given service, which receives the message from AMQ and
+* forwards it to the application.
+"""
 
 from aio_pika import Message
-from aio_pika.abc import AbstractRobustChannel, AbstractRobustQueue, AbstractRobustConnection, AbstractRobustExchange, \
-    AbstractIncomingMessage
+from aio_pika.abc import (
+    AbstractIncomingMessage,
+    AbstractRobustChannel,
+    AbstractRobustConnection,
+    AbstractRobustExchange,
+    AbstractRobustQueue,
+)
 from pika.exchange_type import ExchangeType
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+from amqp.adapter.service_message_factory import ServiceMessageFactory
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import CleverMicroServiceMessage
+from amqp.model.model import ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
-from amqp.router.router_base import RouterBase, CM_REQUEST_EXCHANGE, CM_RESPONSE_EXCHANGE, CM_C_AMQ_QUEUE, \
-    CM_P_RESP_QUEUE, CM_P_REPLY_TO, ANY_RECIPIENT
+from amqp.router.router_base import (
+    ANY_RECIPIENT,
+    CM_C_AMQ_QUEUE,
+    CM_P_REPLY_TO,
+    CM_P_RESP_QUEUE,
+    CM_REQUEST_EXCHANGE,
+    CM_RESPONSE_EXCHANGE,
+    RouterBase,
+)
 
 
 class RouterProducer(RouterBase):
 
     def __init__(self, configuration: AMQConfiguration):
         """
-         * Initialize router - need to supply the config values and RabbitMQ channel.
-         * :param configuration:  adapter's configuration reference.
-         """
+        * Initialize router - need to supply the config values and RabbitMQ channel.
+        * :param configuration:  adapter's configuration reference.
+        """
         super().__init__()
         self.connection: AbstractRobustConnection | None = None
-        self.service_message_factory = CleverMicroServiceMessageFactory(configuration.amq_adapter.service_name)
+        self.service_message_factory = ServiceMessageFactory(
+            configuration.amq_adapter.service_name
+        )
         self.amq_configuration: AMQConfiguration = configuration
         self.channel: AbstractRobustChannel | None = None
         self.cm_request_exchange: AbstractRobustExchange | None = None
         self.cm_response_exchange: AbstractRobustExchange | None = None
 
-        logging.info(f" [*] RouterProducer.<init>: %s, route: %s",
-                     self.amq_configuration.amq_adapter.service_name,
-                     self.amq_configuration.amq_adapter.route_mapping)
+        logging_info(
+            "RouterProducer.<init>: %s, route: %s",
+            self.amq_configuration.amq_adapter.service_name,
+            self.amq_configuration.amq_adapter.route_mapping,
+        )
 
-    async def init_routing_paths(self, channel: AbstractRobustChannel | None):
+    async def init_internal_routing_paths(self, channel: AbstractRobustChannel | None):
+        """
+        * Initialize internal routes for ServiceMessages.
+        * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
+        * created to enable the service discovery.
+        * The Producer mode sends out routing data request on CleverMicroRequest and listens
+        * on CleverMicroResponse for any service responding with its routing data.
         """
-         * Initialize router according to Producer mode.
-         * The logic here sets a producer and listener on CleverMicroRequest/CleverMicroResponse queues
-         * created to enable the service discovery.
-         * The Producer mode sends out routing data request on CleverMicroRequest and listens
-         * on CleverMicroResponse for any service responding with its routing data.
-         """
 
         self.channel = channel
-        logging.info(" [*] RouterProducer.initRoutingPaths, channel open: %s", self.is_open(self.channel))
+        logging_info(
+            "RouterProducer.initRoutingPaths, channel open: %s",
+            self.is_open(self.channel),
+        )
         if self.is_open(self.channel):
             try:
                 # **** set up the service discovery internal exchanges ****
@@ -65,44 +89,64 @@ class RouterProducer(RouterBase):
                     name=CM_RESPONSE_EXCHANGE, type=ExchangeType.fanout
                 )
                 # 2a. declare a Response queue for Routing Data replies from Services
-                cm_response_queue: str = self.get_response_queue_name()
-                response_queue: AbstractRobustQueue = await self.channel.declare_queue(
-                    name=cm_response_queue, durable=True, exclusive=False, auto_delete=False,
-                    arguments={}
+                _cm_response_queue: str = self.get_response_queue_name()
+                _response_queue: AbstractRobustQueue = await self.channel.declare_queue(
+                    name=_cm_response_queue,
+                    durable=True,
+                    exclusive=False,
+                    auto_delete=False,
+                    arguments={},
                 )
                 # 2b. bind queue to CM_RESPONSE_EXCHANGE w/o any conditions (empty routing key)
-                await response_queue.bind(exchange=CM_RESPONSE_EXCHANGE, routing_key="#")
+                await _response_queue.bind(
+                    exchange=CM_RESPONSE_EXCHANGE, routing_key="#"
+                )
                 # 2c. listen for incoming RoutingData messages
-                await response_queue.consume(callback=self.handle_routing_data_message, no_ack=False)
-                logging.info(" [*] Routing master listens for Route updates on %s", cm_response_queue)
+                _c_tag = await _response_queue.consume(
+                    callback=self.handle_routing_data_message, no_ack=False
+                )
+                logging_info(
+                    " [%s] Routing master listens for Route updates on %s",
+                    _c_tag,
+                    _cm_response_queue,
+                )
             except Exception as ioe:
-                logging.error(" [E] RouterProducer FATAL: cannot initialize, %s", ioe)
+                logging_error("RouterProducer FATAL: cannot initialize, %s", ioe)
 
         else:
-            logging.error(" [E] RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open.")
+            logging_error(
+                "RouterProducer FATAL: cannot initialize, RabbitMQ channel is not open."
+            )
 
     async def handle_routing_data_message(self, message: AbstractIncomingMessage):
         """
-         * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
-         * routing data, invokes addRoute or removeRoute depending on the type of the message.
-         """
-        logging.info(f" [x] Received {message.body}, m={message.message_id}, p={message.properties}")
+        * Handler for CLEVER_MICRO_RESPONSE queue messages. Used in Master mode to handle receipt of
+        * routing data, invokes addRoute or removeRoute depending on the type of the message.
         """
-        logging.info(" [DBG] ******[%s] ROUTING_DATA %s >>> %s",
-              delivery.getEnvelope().getDeliveryTag(),
-              delivery.getEnvelope().toString(), str(delivery.getBody()))
-              """
-        # ACK message now so that it is not being redelivered
+        logging_info(
+            "[%s] Received ROUTING DATA: [%s], msgId=%s",
+            message.delivery_tag,
+            message.body,
+            message.message_id,
+        )
         await message.ack(multiple=False)
-        cm_message: CleverMicroServiceMessage = self.service_message_factory.from_bytes(message.body)
+        cm_message: ServiceMessage = self.service_message_factory.from_bytes(
+            message.body
+        )
         # ignore the message if it comes from ourselves (even if from different instance)
-        logging.info(" [DBG] ******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
-                     message.delivery_tag,
-                     self.service_message_factory.to_string(cm_message),
-                     self.amq_configuration.amq_adapter.route_mapping,
-                     self.amq_configuration.amq_adapter.service_name,
-                     cm_message.reply_to)
-        if cm_message.is_valid() and not self.amq_configuration.amq_adapter.service_name == cm_message.reply_to:
+        logging_debug(
+            "******[%s] ROUTING_DATA, msg=%s, routeMap=[%s] sn=%s, replyTo=%s",
+            message.delivery_tag,
+            self.service_message_factory.to_string(cm_message),
+            self.amq_configuration.amq_adapter.route_mapping,
+            self.amq_configuration.amq_adapter.service_name,
+            cm_message.reply_to,
+        )
+        if (
+            cm_message.is_valid()
+            and not self.amq_configuration.amq_adapter.service_name
+            == cm_message.reply_to
+        ):
             if cm_message.message_type == ServiceMessageType.ROUTING_DATA:
                 self.add_routes(cm_message.message)
             if cm_message.message_type == ServiceMessageType.ROUTING_DATA_REMOVE:
@@ -110,72 +154,83 @@ class RouterProducer(RouterBase):
 
     async def request_routes_from_adapters(self):
         """
-         * Send a broadcast message prompting all listening adapters to reply with own routing data.
-         """
-        logging.info(" [*] RouterProducer.requestRoutesFromAdapters, queue name: %s", CM_REQUEST_EXCHANGE)
+        * Send a broadcast message prompting all listening adapters to reply with own routing data.
+        """
+        logging_info(
+            "RouterProducer.requestRoutesFromAdapters, queue name: %s",
+            CM_REQUEST_EXCHANGE,
+        )
         try:
-            serviceMessage: CleverMicroServiceMessage = self.service_message_factory.of(
+            serviceMessage: ServiceMessage = self.service_message_factory.of(
                 ServiceMessageType.ROUTING_DATA_REQUEST, "", ANY_RECIPIENT
             )
-            if not await self.publish_service_message(serviceMessage, self.cm_request_exchange):
-                logging.warning(" [E] RouterProducer could not request route data: %s channel is not open.",
-                                CM_REQUEST_EXCHANGE)
+            if not await self.publish_service_message(
+                serviceMessage, self.cm_request_exchange
+            ):
+                logging_warning(
+                    "RouterProducer could not request route data: %s channel is not open.",
+                    CM_REQUEST_EXCHANGE,
+                )
 
         except Exception as ioe:
-            logging.error(" [E] RouterProducer failed request routing data: %s", ioe)
+            logging_error("RouterProducer failed request routing data: %s", ioe)
 
-    async def publish_service_message(self,
-                                      message: CleverMicroServiceMessage,
-                                      exchange: AbstractRobustExchange) -> bool:
+    async def publish_service_message(
+        self, message: ServiceMessage, exchange: AbstractRobustExchange
+    ) -> bool:
+        """
+        * Publishes a service message to the service queue and logs success log entry.
+        *
+        * :param message:   Service Message to be sent
+        * :param exchange: target exchange name where to publish the message
+        * :return True if channel is open, False otherwise
+        * @throws IOException if something else goes wrong
         """
-         * Publishes a service message to the service queue and logs success log entry.
-         *
-         * :param message:   Service Message to be sent
-         * :param exchange: target exchange name where to publish the message
-         * :return True if channel is open, False otherwise
-         * @throws IOException if something else goes wrong
-         """
         if self.is_open(self.channel):
             binary_content: bytes = self.service_message_factory.serialize(message)
             pika_message: Message = Message(
                 body=binary_content,
-                content_encoding='utf-8',
+                content_encoding="utf-8",
                 delivery_mode=2,
                 content_type="application/octet-stream",
                 headers=None,
                 priority=0,
-                correlation_id=None
+                correlation_id=None,
             )
             await exchange.publish(message=pika_message, routing_key="")
-            logging.info(" [x] Service Message Sent to %s, msg: %s", exchange.name, str(binary_content))
+            logging_info(
+                "Service Message Published to %s, msg: %s",
+                exchange.name,
+                str(binary_content),
+            )
             return True
         return False
 
     def get_reply_to_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_REPLY_TO
 
     def get_response_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_P_RESP_QUEUE
 
     def get_consume_queue_name(self) -> str:
         """
-         * self should create a unique name for self process and also when there's several instances.
-         * @return unique reply-to queue name.
-         """
+        * self should create a unique name for self process and also when there's several instances.
+        * @return unique reply-to queue name.
+        """
         return self.amq_configuration.amq_adapter.adapter_prefix() + CM_C_AMQ_QUEUE
 
     def is_open(self, _channel: AbstractRobustChannel) -> bool:
         """
-         * Test if the channel is usable (opened).
-         * :param _channel: the channel
-         * :return True if the channel is opened.
-         """
+        * Test if the channel is usable (opened).
+        * :param _channel: the channel
+        * :return True if the channel is opened.
+        """
         return _channel is not None and not _channel.is_closed

amqp/router/utils.py

@@ -1,19 +1,37 @@
+import asyncio
 import re
 
 pattern = re.compile(r"[^a-zA-Z0-9.#/\-]")
+AWAIT_SLEEP = 0.05  # seconds
 
 
 def sanitize_as_rabbitmq_name(input_str: str) -> str:
     matcher = pattern.search(input_str)
-    token = input_str[:matcher.start()] if matcher else input_str
+    token = input_str[: matcher.start()] if matcher else input_str
     return sanitize_as_rabbitmq_domain_name(token)
 
 
 def sanitize_as_rabbitmq_domain_name(token: str) -> str:
     # Step 1: Replace non-alphanumeric characters (except periods) with a period
-    step1 = re.sub(r'[^A-Za-z0-9.#]', '.', token)
+    step1 = re.sub(r"[^A-Za-z0-9.#]", ".", token)
     # Step 2: Replace multiple consecutive periods with a single period
-    step2 = re.sub(r'\.+', '.', step1)
+    step2 = re.sub(r"\.+", ".", step1)
     # Step 3: Remove trailing '.' character, if applicable
     step_last_char = len(step2) - 1
-    return step2[:-1] if step_last_char > 0 and step2[step_last_char] == '.' else step2
+    return step2[:-1] if step_last_char > 0 and step2[step_last_char] == "." else step2
+
+
+async def await_future(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete.
+    """
+    while not future.done():
+        await asyncio.sleep(sleep_time)
+
+
+async def await_result(future: asyncio.Future, sleep_time=AWAIT_SLEEP):
+    """
+    Wait for a future to complete and return its result.
+    """
+    await await_future(future, sleep_time=sleep_time)
+    return future.result()

amqp/service/amq_message_handler.py

@@ -0,0 +1,293 @@
+import asyncio
+import os
+import threading
+import time
+from asyncio import AbstractEventLoop, Future
+from typing import Dict
+
+from aio_pika import Message
+from aio_pika.abc import AbstractIncomingMessage, AbstractRobustExchange
+from opentelemetry import trace
+from opentelemetry.trace import Tracer, TraceState
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.cleverthis_service_adapter import AMQMessage, CleverThisServiceAdapter
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
+from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
+from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.adapter.serializer import map_as_string
+from amqp.adapter.service_message_factory import ServiceMessageFactory
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import (
+    DataMessage,
+    DataResponse,
+    MultipartDataMessage,
+)
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+
+
+def collect_trace_info():
+    trace_map = {}
+    TraceContextTextMapPropagator().inject(
+        carrier=trace_map, context=trace.context_api.get_current()
+    )
+    return trace_map
+
+
+def get_default_trace_state():
+    return TraceState(entries={})
+
+
+def set_text(carrier, key, value):
+    carrier[key] = value
+
+
+class DataMessageHandler:
+    def __init__(
+        self,
+        service_adapter: CleverThisServiceAdapter,
+        tracer: Tracer,
+        message_factory: DataMessageFactory,
+        service_message_factory: ServiceMessageFactory,
+        reply_to_exchange: AbstractRobustExchange,
+        rabbit_mq_client: RabbitMQClient,
+        amq_configuration: AMQConfiguration,
+        loop,
+        backpressure_handler: BackpressureHandler,
+        file_handler: FileHandler = StreamingFileHandler(),
+    ):
+        self.service_adapter: CleverThisServiceAdapter = service_adapter
+        self.tracer = tracer
+        self.message_factory: DataMessageFactory = message_factory
+        self.service_message_factory: ServiceMessageFactory = service_message_factory
+        self.reply_to_exchange: AbstractRobustExchange = reply_to_exchange
+        self.rabbit_mq_client: RabbitMQClient = rabbit_mq_client
+        self.rabbit_mq_url = amq_configuration.dispatch.rabbit_mq_url
+        self.output_dir = amq_configuration.dispatch.download_dir
+        self.loop = loop
+        self.service_adapter.loop = loop
+        # Dictionary to store outstanding Futures
+        self.outstanding: Dict[str, asyncio.Future] = {}
+        self.reply_to: str | None = None
+        self.file_handler: FileHandler = file_handler
+        self.backpressure_handler: BackpressureHandler = backpressure_handler
+
+    async def inbound_data_message_callback_as_thread(
+        self, message: AbstractIncomingMessage
+    ):
+        """
+        The callback method that is called when a DataMessage is received from RabbitMQ, starts new Thread.
+        """
+        threading.Thread(
+            target=lambda: asyncio.run(
+                self.inbound_data_message_callback_no_thread(message)
+            )
+        ).start()
+
+    async def inbound_data_message_callback_no_thread(
+        self, message: AbstractIncomingMessage
+    ):
+        """
+        Receives the DataMessage as bytes from RabbitMQ queue and deserializes it to the DataMessage object
+        Ensures Jaeger trace-id propagation.
+        Invokes custom handler/mapper/adapter method that corresponds to this message
+        :param message:
+        :return:
+        """
+        start = time.time()
+        amq_message = await self.reconstitute_data_message(message)
+        if amq_message is not None:
+            self.ensure_trace_info(amq_message)
+            if amq_message.magic == DataMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
+                await self.run_http_request_magic(message, amq_message, self.loop)
+            else:
+                logging_error(
+                    "Unknown MAGIC value: [%s], don't know how to interpret the message",
+                    amq_message.magic,
+                )
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), self.loop)
+                return
+        else:
+            logging_error(
+                "######[%s] No valid AMQ Message present.", message.delivery_tag
+            )
+        self.record_duration(start, message.delivery_tag)
+        asyncio.run_coroutine_threadsafe(message.ack(multiple=False), self.loop)
+
+    async def run_http_request_magic(
+        self,
+        message: AbstractIncomingMessage,
+        amq_message: DataMessage,
+        loop: AbstractEventLoop,
+    ) -> None:
+        """
+        Synchronous version of on_message method. This is a wrapper around the async on_message method.
+        It also counts parallel executions and handles backpressure.
+        :param message: RabbitMQ raw message
+        :param amq_message: DataMessage
+        :param loop: Event loop
+        :return: DataResponse
+        """
+        # Increment the counter for parallel executions
+        self.backpressure_handler.increase_parallel_executions()
+        logging_info(
+            f"PARALLEL: Current Capacity [{self.backpressure_handler.current_parallel_executions}] of [{self.backpressure_handler.parallel_workers}]"
+        )
+        if message.reply_to:
+            self.reply_to = message.reply_to
+
+        try:
+            await self.backpressure_handler.check_overload_condition()
+            _a_message = AMQMessage(amq_message, self.rabbit_mq_client.connection)
+            _response: DataResponse = await self.service_adapter.on_message(_a_message)
+            logging_debug(f"Result in thread: {_response}")
+            try:
+                await self.send_reply(message.reply_to, _response)
+            except Exception as e:
+                logging_error(f"Processing message: {e}")
+                asyncio.run_coroutine_threadsafe(message.nack(requeue=False), loop=loop)
+
+            if isinstance(amq_message, MultipartDataMessage):
+                # remove the downloaded files from the output_dir
+                for file_id, filename in amq_message.extra_data.items():
+                    try:
+                        logging_info(
+                            f"[{message.delivery_tag}] Deleting file: {file_id} in {filename}"
+                        )
+                        os.remove(filename)
+                    except OSError as e:
+                        logging_error(f"Deleting file {filename}: {e}")
+        except Exception as e:
+            logging_error(f"Request handler: {e}")
+
+        self.backpressure_handler.decrease_parallel_executions()
+
+    async def reconstitute_data_message(
+        self, message: AbstractIncomingMessage
+    ) -> DataMessage | None:
+        amq_message: DataMessage | None = None
+        delivery_tag = message.delivery_tag
+        addressing: int = message.headers.get("clevermicro.addressing", 0)
+        if addressing == 0:
+            amq_message = DataMessageFactory.from_bytes(message.body)
+        else:
+            amq_message = await DataMessageFactory.from_stream(
+                message.body, rabbitmq_url=self.rabbit_mq_url, loop=self.loop
+            )
+            logging_info(
+                f"######[{delivery_tag}] AMQ Message from stream: {amq_message}"
+            )
+            if amq_message is not None:
+                extra_data = await self.file_handler.on_message(
+                    message=message,
+                    json_data=amq_message.body(),
+                    rabbitmq_url=self.rabbit_mq_url,
+                    loop=self.loop,
+                    output_dir=self.output_dir,
+                )
+                logging_info(
+                    f"######[{delivery_tag}] Multipart download result: {extra_data}"
+                )
+                amq_message = MultipartDataMessage(amq_message, extra_data)
+        if amq_message:
+            logging_info(
+                "######[%s] AMQ Message received, ctag=%s, msgId=%s, %s:%s, traceInfo=%s",
+                message.delivery_tag,
+                message.consumer_tag,
+                amq_message.id,
+                amq_message.method,
+                amq_message.path,
+                map_as_string(amq_message.trace_info),
+            )
+        return amq_message
+
+    def ensure_trace_info(self, amq_message: DataMessage):
+        propagator = TraceContextTextMapPropagator()
+        context = propagator.extract(
+            carrier=amq_message.trace_info, context=trace.context_api.get_current()
+        )
+        self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
+        # logging_info("CTX=%s", context)
+        # current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
+        # context_with_span = trace.set_span_in_context(current_span, context)
+        # logging.info(" [*] CTX2=%s", context_with_span)
+        # propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
+        # logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
+        #             map_as_string(amq_message.trace_info), context_with_span, context)
+
+    def record_duration(self, start, delivery_tag):
+        global last_data_message_time
+        last_data_message_time = time.time()
+        logging_info(
+            "######[%s] Done, single ACK: %sms.",
+            delivery_tag,
+            last_data_message_time - start,
+        )
+
+    async def send_reply(self, reply_to: str, response: DataResponse):
+        """
+        The Service side code. When the CleverXXX service has output ready, in DataResponse object,
+        call this method to return the reply back to the caller
+        :param reply_to:connection.channel
+        :param response:
+        :return:
+        """
+        if reply_to:
+            pika_message: Message = Message(
+                body=DataResponseFactory.serialize(response),
+                correlation_id=response.id,
+                content_type=(
+                    response.content_type[0]
+                    if isinstance(response.content_type, list)
+                    else response.content_type
+                ),
+            )
+            _res = asyncio.run_coroutine_threadsafe(
+                self.reply_to_exchange.publish(
+                    message=pika_message, routing_key=reply_to
+                ),
+                self.loop,
+            )
+            logging_debug(
+                "###### DONE Reply Published to exchg:(%s) To=%s, res=%s",
+                self.reply_to_exchange.name,
+                reply_to,
+                _res,
+            )
+
+    async def reply_received_callback(self, message: AbstractIncomingMessage):
+        """
+        The reply for DataMessage that we sent out earlier - must match the received response with
+        the original request and respond to the caller.
+        :param message:
+        :return:
+        """
+        reply_id = message.correlation_id
+        delivery_tag = message.delivery_tag
+        debug = f"######[{delivery_tag}] (RPC REPLY) env: {message.headers}, props: {message.properties}, BODY: {message.body.decode()}"
+        logging_debug(debug)
+        # TODO - figure out implementation of this part, as this is sending reply back either to a service,
+        #        or to the user
+        logging_info(
+            f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}"
+        )
+
+        future: Future = self.outstanding.pop(reply_id, None)
+        # if reply is None:
+        # logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
+        #             delivery_tag, reply_id)
+        # else:
+        # result = DataResponseFactory.from_bytes(body)
+        # reply.tracing_span().end()
+        # TODO - figure out the right response format (not needed for Clever Swarm v0.1)
+        # that currently runs Java version, this method here is for the future compatibility to enable
+        # internal, service-2-service communication, and the proper response type will be decided than
+        # reply.response().set_result(Response.ok(result.body, content_type).build())
+        if future:
+            response: DataResponse = DataResponseFactory.from_bytes(message.body)
+            # Complete the Future with the response data
+            future.set_result(response.body())
+
+        await message.ack(multiple=False)

amqp/service/amq_service.py

@@ -3,105 +3,126 @@ import logging.config
 import os
 import asyncio
 from asyncio import Future
+from threading import Thread
+from typing import Optional
 
 from aio_pika.abc import AbstractRobustExchange
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.backpressure_handler import BackpressureHandler
+from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.adapter.amq_route_factory import AMQRouteFactory
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
+from amqp.adapter.logging_utils import logging_info, logging_warning
+from amqp.adapter.service_message_factory import ServiceMessageFactory
 
 from amqp.config.amq_configuration import AMQConfiguration
-from amqp.model.model import AMQMessage
-from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
-from amqp.service.service_message_handler import AMQServiceMessageHandler
+from amqp.model.model import DataMessage
+from amqp.rabbitmq.rabbit_mq_client import RabbitMQClient
+from amqp.service.amq_message_handler import DataMessageHandler
 from amqp.service.tracing import initialize_trace
 
 
 logging.basicConfig(level=logging.DEBUG)
 logging.info(f"CWD = {os.getcwd()}")
-logging_conf_path = os.path.join(os.getcwd(), 'amqp', 'service', 'logging.conf')
+logging_conf_path = os.path.join(os.getcwd(), "amqp", "service", "logging.conf")
 if os.path.exists(logging_conf_path):
     logging.config.fileConfig(logging_conf_path)
 else:
-    logging_conf_path = os.path.join(os.getcwd(), 'logging.conf')
+    logging_conf_path = os.path.join(os.getcwd(), "logging.conf")
     if os.path.exists(logging_conf_path):
         logging.config.fileConfig(logging_conf_path)
 
+
 class AMQService:
     CLEVER_AMQP_CLIENT_VERSION = os.getenv("CLEVER_AMQP_CLIENT_VERSION")
     MAX_WAIT = 300  # defaults to 5 minutes (300 seconds)
 
-    def __init__(self, amq_configuration: AMQConfiguration, service_adapter):
+    def __init__(self, amq_configuration: AMQConfiguration, service_adapter=None):
         logging.info("***********************************************")
         logging.info("**********  AMQ   Service   *******************")
         logging.info("***********************************************")
-        loop = asyncio.get_event_loop()
+        self.loop = asyncio.new_event_loop()
 
         self.amq_configuration = amq_configuration
-        self.message_factory = AMQMessageFactory.get_instance(self.amq_configuration.amq_adapter.generator_id)
-        self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
+        self.service_adapter = service_adapter
+        self.data_message_factory = DataMessageFactory.get_instance(
+            self.amq_configuration.amq_adapter.generator_id
+        )
+        self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
         self.tracer = initialize_trace(name=amq_configuration.amq_adapter.service_name)
-        self.amq_service_message_handler = None
-        self.service_message_factory = CleverMicroServiceMessageFactory(
+        self.amq_data_message_handler = None
+        self.service_message_factory = ServiceMessageFactory(
             self.amq_configuration.amq_adapter.service_name
         )
+        self.backpressure_thread: Optional[Thread] = None
+
+    def run(self):
         # self.init(loop) will initialize RabbitMQ connection
-        loop.run_until_complete(self.init(loop, service_adapter))
-        # loop.run_until_complete(consume(loop))
-        # self.consumer_thread = asyncio.create_task(self.rabbit_mq_consumer.channel.start_consuming())
-        # self.rabbit_mq_consumer.request_routes()
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_until_complete(self.init(self.loop, self.service_adapter))
 
         logging.info("***********************************************")
         logging.info("******  AMQ   Service  Init  Complete *********")
         logging.info("***********************************************")
-        loop.run_forever()
+        self.loop.run_forever()
 
     async def init(self, loop, service_adapter):
-        exchange: AbstractRobustExchange = await self.rabbit_mq_consumer.init(loop)
-        self.amq_service_message_handler = AMQServiceMessageHandler(
-            service_adapter, self.tracer, self.message_factory, exchange
+        _reply_to_exchange: AbstractRobustExchange = await self.rabbit_mq_client.init(
+            loop
+        )
+        backpressure_handler = BackpressureHandler(
+            channel=self.rabbit_mq_client.channel,
+            loop=loop,
+            config=self.amq_configuration,
+        )
+        self.amq_data_message_handler = DataMessageHandler(
+            service_adapter,
+            self.tracer,
+            self.data_message_factory,
+            self.service_message_factory,
+            _reply_to_exchange,
+            self.rabbit_mq_client,
+            self.amq_configuration,
+            loop=loop,
+            backpressure_handler=backpressure_handler,
         )
         await self.register_routes()
-        await self.rabbit_mq_consumer.request_routes()
+        await self.rabbit_mq_client.request_routes()
+        self.backpressure_thread = backpressure_handler.start_backpressure_monitor()
 
     def cleanup(self):
-        logging.info(" [DBG] AMQService.cleanup() rmqc=%s", self.rabbit_mq_consumer)
-        if self.rabbit_mq_consumer:
-            self.rabbit_mq_consumer.cleanup()
+        logging_info("AMQService.cleanup() rmqc=%s", self.rabbit_mq_client)
+        if self.rabbit_mq_client:
+            self.rabbit_mq_client.cleanup()
 
     async def reinitialize_if_disconnected(self):
-        if not self.rabbit_mq_consumer.channel or self.rabbit_mq_consumer.channel.is_closed:
-            self.rabbit_mq_consumer = RabbitMQConsumer(self.amq_configuration)
-            await self.rabbit_mq_consumer.init(loop=None)
+        if not self.rabbit_mq_client.channel or self.rabbit_mq_client.channel.is_closed:
+            self.rabbit_mq_client = RabbitMQClient(self.amq_configuration)
+            await self.rabbit_mq_client.init(loop=self.loop)
             await self.register_routes()
-            await self.rabbit_mq_consumer.request_routes()
+            await self.rabbit_mq_client.request_routes()
 
     async def register_routes(self) -> AbstractRobustExchange | None:
         route_mapping = self.amq_configuration.amq_adapter.route_mapping
         if AMQRouteFactory.validate(route_mapping):
             try:
-                await self.rabbit_mq_consumer.register_inbound_routes(
-                    route_mapping, self.amq_service_message_handler.inbound_message_callback
+                await self.rabbit_mq_client.register_inbound_routes(
+                    route_mapping,
+                    self.amq_data_message_handler.inbound_data_message_callback_no_thread,
                 )
-                return await self.rabbit_mq_consumer.register_rpc_handler(
-                    self.amq_service_message_handler.reply_received_callback
+                return await self.rabbit_mq_client.register_data_reply_callback(
+                    self.amq_data_message_handler.reply_received_callback
                 )
             except Exception as e:
                 raise RuntimeError(e)
         elif route_mapping:
-            logging.info(" [W] route [%s] failed validation, no routes were registered.", route_mapping)
+            logging_warning(
+                "route [%s] failed validation, no routes were registered.",
+                route_mapping,
+            )
         return None
 
-    def send_message(self, message: AMQMessage, future: Future):
-        self.amq_service_message_handler.outstanding[str(message.id)] = future
+    def send_message(self, message: DataMessage, future: Future):
+        self.amq_data_message_handler.outstanding[str(message.id)] = future
 
     def remove_outstanding_future(self, message_id: str):
-        self.amq_service_message_handler.outstanding.pop(message_id, None)
-
-
-# if __name__ == "__main__":
-    # amq_configuration: AMQConfiguration = AMQConfiguration('../config/application.properties.local')
-    # amq_service: AMQService = AMQService(amq_configuration, CleverSwarmAmqpAdapter(amq_configuration))
-    # amq_service.rabbit_mq_consumer.channel.start_consuming()
-    # amq_service.consumer_thread.cancel()
-    # amq_service.rabbit_mq_consumer.connection.close()
+        self.amq_data_message_handler.outstanding.pop(message_id, None)

amqp/service/multipart_parser.py

@@ -1,50 +0,0 @@
-import io
-import json
-import logging
-
-import python_multipart
-from fastapi import UploadFile
-from python_multipart import multipart
-from python_multipart.multipart import Field, File
-
-from amqp.model.model import AMQMessage
-
-
-class CleverMultiPartParser:
-    def __init__(self, message: AMQMessage):
-        self.form_data = {}
-        self.is_multipart = False
-        self.is_json = False
-        self.is_valid = True
-        # Convert each list of strings to a single string joined by newline, then to bytes
-        headers = {key: "\n".join(value).encode("utf-8") for key, value in message.headers.items()}
-        content_type: str | bytes | None = headers.get("Content-Type")
-        content_type, params = multipart.parse_options_header(content_type)
-        if content_type.decode('utf-8') in ["application/octet-stream",
-                            "multipart/form-data",
-                            "application/x-www-form-urlencoded",
-                            "application/x-url-encoded"]:
-            try:
-                python_multipart.parse_form(headers, io.BytesIO(message.body()), self.on_field, self.on_file)
-                self.is_multipart = True
-            except Exception as e:
-                logging.error(f"Error parsing multipart/form-data: {e}")
-                try:
-                    self.form_data = json.loads(message.body())
-                    self.is_json = True
-                except Exception as jsonerror:
-                    logging.error(f"Error parsing JSON: {jsonerror}")
-                    self.is_valid = False
-        else:
-            self.form_data = message.body()
-
-    def on_field(self, field: Field):
-        print(field)
-        self.form_data[field.field_name.decode('utf-8')] = field.value
-        print(self.form_data)
-
-    def on_file(self, file: File):
-        print(file)
-        self.form_data[file.field_name.decode('utf-8')] = \
-            UploadFile(file.file_object, size=file.size, filename=file.file_name)
-        print(self.form_data)

amqp/service/service_message_handler.py

@@ -1,186 +0,0 @@
-import asyncio
-import logging
-import time
-import concurrent.futures
-import os
-from asyncio import Future
-from typing import Dict
-
-from aio_pika import IncomingMessage, Message
-from aio_pika.abc import AbstractIncomingMessage, AbstractRobustExchange
-
-from opentelemetry import trace
-from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
-from opentelemetry.trace import Tracer, TraceState
-
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.adapter.amq_response_factory import AMQResponseFactory
-from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter
-from amqp.model.model import AMQResponse, AMQMessage
-from amqp.router.serializer import map_as_string
-
-parallel_workers = os.getenv("PARALLEL_WORKERS", default=10)
-
-# Shared thread pool for parallel execution
-executor = concurrent.futures.ThreadPoolExecutor(max_workers=parallel_workers)
-
-def collect_trace_info():
-    trace_map = {}
-    TraceContextTextMapPropagator().inject(
-        carrier=trace_map,
-        context=trace.context_api.get_current()
-    )
-    return trace_map
-
-
-def get_default_trace_state():
-    return TraceState(entries={})
-
-
-def set_text(carrier, key, value):
-    carrier[key] = value
-
-
-class AMQServiceMessageHandler:
-    def __init__(self, service_adapter: CleverThisServiceAdapter,
-                 tracer: Tracer, message_factory: AMQMessageFactory,
-                 reply_to_exchange: AbstractRobustExchange):
-        self.service_adapter: CleverThisServiceAdapter = service_adapter
-        self.tracer = tracer
-        self.message_factory: AMQMessageFactory = message_factory
-        self.reply_to_exchange: AbstractRobustExchange = reply_to_exchange
-        # Dictionary to store outstanding Futures
-        self.outstanding: Dict[str, asyncio.Future] = {}
-
-    async def inbound_message_callback(self, message: AbstractIncomingMessage):
-        """
-        Receives the message as bytes from RabbitMQ queue and deserializes it to the AMQMessage
-        Ensures Jaeger trace-id propagation.
-        Invokes custom handler/mapper/adapter method that corresponds to this message
-        :param message:
-        :return:
-        """
-        start = time.time()
-        delivery_tag = message.delivery_tag
-        debug = f" [DBG] ######[{delivery_tag}] (AMQ MESSAGE), env: {message.headers}, props: {message.properties}"
-        logging.info(debug)
-
-        amq_message: AMQMessage = AMQMessageFactory.from_bytes(message.body)
-        logging.info(" [*] ######[%s] AMQ Message received, ctag=%s, msgId=%s, traceInfo=%s",
-                     delivery_tag, message.consumer_tag, amq_message.id, map_as_string(amq_message.trace_info))
-
-        propagator = TraceContextTextMapPropagator()
-        context = propagator.extract(carrier=amq_message.trace_info, context=trace.context_api.get_current())
-        self.tracer.start_span(name="AMQ-Adapter-Consumer", context=context)
-
-        logging.info(" [*] CTX=%s", context)
-        # current_span = self.tracer.start_span("AMQ-Adapter.Processing.Request", parent=parent_span)
-        # context_with_span = trace.set_span_in_context(current_span, context)
-        # logging.info(" [*] CTX2=%s", context_with_span)
-        # propagator.inject(context_with_span, amq_message.trace_info, self.set_text)
-        # logging.info(" [***] NEW TI=%s CTX=%s SCOPE=%s",
-        #             map_as_string(amq_message.trace_info), context_with_span, context)
-
-        if amq_message.magic == AMQMessageFactory.MAGIC_BYTE_HTTP_REQUEST:
-            future = executor.submit(self.service_adapter.sync_on_message, amq_message)
-            # response: AMQResponse = asyncio.run(self.http_adapter.on_message(amq_message))
-            # Wait for the result and send it back to the reply queue
-            try:
-                response: AMQResponse = future.result()  # Block until the result is available
-                await self.send_reply(message.reply_to, delivery_tag, response)
-            except Exception as e:
-                print(f"[E] Error processing message: {e}")
-                await message.nack(requeue=False)
-                return
-        else:
-            logging.error(" [E] Unknown MAGIC value: [%s], don't know how to interpret the message",
-                          amq_message.magic)
-            await message.nack(requeue=False)
-            return
-
-        duration = time.time() - start
-        logging.info(" [x] ######[%s] Done, single ACK: %sms.", delivery_tag, duration)
-        await message.ack(multiple=False)
-
-    def on_http_response(self, delivery, deliveryTag, channel, http_response):
-        """
-        In loose coupling mode, receives the HTTP response to the REST request the HTTPAdapter made to the
-        associated CleverXXX service. Convert the response to AMQResponse message and use
-        RabbitMQ RPC reply-to mechanism to deliver this response back to API Gateway side AMQProducer.
-        :param delivery:
-        :param deliveryTag:
-        :param channel:
-        :param http_response:
-        :return:
-        """
-        response = AMQResponse(
-            delivery.properties.correlation_id,
-            http_response.status_code,
-            http_response.headers.get("Content-Type"),
-            http_response.body,
-            http_response.status_message,
-            "",
-            collect_trace_info()
-        )
-        try:
-            self.send_reply(delivery.properties.reply_to, deliveryTag, response)
-        except Exception as e:
-            raise RuntimeError(e)
-
-    async def send_reply(self, reply_to: str, delivery_tag: int | None, response: AMQResponse):
-        """
-        The Service side code. When the CleverXXX service has output ready, in AMQResponse object,
-        call this method to return the reply back to the caller
-        :param reply_to:
-        :param delivery_tag:
-        :param response:
-        :return:
-        """
-        if reply_to:
-            logging.info(" [*] ######[%s] AMQ Message Reply, To=%s, msgId=%s",
-                         delivery_tag, reply_to, response.id)
-            pika_message: Message = Message(
-                body=AMQResponseFactory.serialize(response),
-                correlation_id=response.id,
-                content_type=response.content_type[0]
-                if isinstance(response.content_type, list) else response.content_type
-            )
-            await self.reply_to_exchange.publish(
-                message=pika_message,
-                routing_key=reply_to
-            )
-
-    async def reply_received_callback(self, message: IncomingMessage):
-        """
-        The producer/client (API Gateway) facing response_message recipient - must match the received response with
-        the original request and produce the HTTP reply.
-        Note: this code runs on the API Gateway side of RabbitMQ
-        :param message:
-        :return:
-        """
-        reply_id = message.correlation_id
-        content_type = message.content_type
-        delivery_tag = message.delivery_tag
-        debug = f" [DBG] ######[{delivery_tag}] (RPC REPLY) env: {message.headers}, props: {message.properties}, BODY: {message.body.decode()}"
-        logging.info(debug)
-        # TODO - figure out implementation of this part, as this is sending reply back either to a service,
-        #        or to the user
-        logging.info(f"OUTSTANDING RECV: {self.outstanding}, size={len(self.outstanding)}")
-
-        future: Future = self.outstanding.pop(reply_id, None)
-        # if reply is None:
-        # logging.warn(" [W] ######[%s] IGNORING REPLY. No outstanding request for ID %s",
-        #             delivery_tag, reply_id)
-        # else:
-        # result = AMQResponseFactory.from_bytes(body)
-        # reply.tracing_span().end()
-        # TODO - figure out the right response format (not needed for Clever Swarm v0.1)
-        # that currently runs Java version, this method here is for the future compatibility to enable
-        # internal, service-2-service communication, and the proper response type will be decided than
-        # reply.response().set_result(Response.ok(result.body, content_type).build())
-        if future:
-            response: AMQResponse = AMQResponseFactory.from_bytes(message.body)
-            # Complete the Future with the response data
-            future.set_result(response.body())
-
-        await message.ack(multiple=False)

amqp/service/stream_parser.py

@@ -1,47 +0,0 @@
-import json
-import struct
-import io
-import pika
-
-class StreamParser:
-    def __init__(self):
-        self.buffer = io.BytesIO()  # Buffer to accumulate chunks
-        self.metadata_length = None  # Length of the metadata
-        self.metadata = None  # Parsed JSON metadata
-        self.file_writer = None  # File writer for the remaining data
-
-    def process_chunk(self, chunk):
-        """
-        Process a chunk of data and handle it according to the rules.
-        """
-        self.buffer.write(chunk)
-
-        # If metadata length is not yet known, try to read it
-        if self.metadata_length is None and self.buffer.tell() >= 4:
-            self.buffer.seek(0)
-            self.metadata_length = struct.unpack('>I', self.buffer.read(4))[0]
-
-        # If metadata length is known but metadata is not yet parsed, try to parse it
-        if self.metadata_length is not None and self.metadata is None:
-            if self.buffer.tell() >= 4 + self.metadata_length:
-                self.buffer.seek(4)
-                metadata_bytes = self.buffer.read(self.metadata_length)
-                self.metadata = json.loads(metadata_bytes.decode('utf-8'))
-                print("Metadata:", self.metadata)
-
-                # Prepare to write the remaining data to a file
-                self.file_writer = open(self.metadata.get("filename", "output_file"), "wb")
-
-        # If metadata is parsed, write the remaining data to the file
-        if self.metadata is not None:
-            remaining_data = self.buffer.read()
-            if remaining_data:
-                self.file_writer.write(remaining_data)
-
-    def close(self):
-        """
-        Close the file writer and clean up.
-        """
-        if self.file_writer:
-            self.file_writer.close()
-        self.buffer.close()

amqp/service/tracing.py

@@ -11,13 +11,17 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 import os
 
 
-def initialize_trace(app = None, name = None) -> Tracer:
+def initialize_trace(app=None, name=None) -> Tracer:
     # Resource can be required for some backends, e.g. Jaeger or Prometheus
     # If resource wouldn't be set - traces wouldn't appear in Jaeger.
     # The 'name' value is the name shown in Jaeger.
-    resource = Resource(attributes={
-        SERVICE_NAME: name if name else os.environ.get('APPLICATION_NAME', 'Python App')
-    })
+    resource = Resource(
+        attributes={
+            SERVICE_NAME: (
+                name if name else os.environ.get("APPLICATION_NAME", "Python App")
+            )
+        }
+    )
     #
     # 1. Tracing (Jaeger) configuration
     # ----------------------------------
@@ -82,7 +86,6 @@ def initialize_trace(app = None, name = None) -> Tracer:
     #    of that parent span, indicating visually the progression of requests via different services
     #    for as long as the current span is provided in the traceparent header to next service.
 
-
     # 2. Metrics (Prometheus) configuration
     # -------------------------------------
     #
@@ -97,4 +100,3 @@ def initialize_trace(app = None, name = None) -> Tracer:
     metrics.set_meter_provider(provider)
 
     return trace.get_tracer(name)
-

cleverswarm/clever_swarm_adapter.py

@@ -0,0 +1,304 @@
+"""
+This module provides the CleverSwarm AMQP Adapter for handling API requests via AMQP.
+The code IS MEANT to be part of CleverSwarm codebase, and has direct dependencies on the CleverSwarm codebase.
+It is NOT MEANT to be used as a standalone module. It WILL NOT run outside the CleverSwarm endpoints context.
+"""
+
+import importlib
+import pathlib
+import re
+from collections import defaultdict
+from threading import Thread
+from typing import Optional, List, Dict
+from aiohttp import ClientSession
+from fastapi.routing import APIRoute
+from fastapi.security import OAuth2PasswordRequestForm
+from starlette.datastructures import UploadFile, Headers
+
+# CleverSwarm specific imports
+import core.deps
+from rest.v0 import api
+from schemas.user_schema import UserSchema
+
+from amqp.adapter.data_parser import AMQDataParser
+
+# AMQP specific imports
+from amqp.adapter.data_response_factory import AMQResponseFactory
+from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, AMQMessage
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import AMQResponse, AMQErrorMessage
+from amqp.service.amq_service import AMQService
+
+
+# Suffix for the method name with serialization wrapper, prefer use this one over the orig endpoint
+PREFERRED_SUFFIX = "_json"
+
+
+def clone_and_adapt_route(original_route: APIRoute) -> APIRoute:
+    """
+    Creates a clone of an APIRoute. It checks if there's preferred endpoint with json serialization.
+    It also adjusts the path regex to ensure it matches the path in different API versions.
+    """
+
+    # Check if the route has a preferred endpoint with json serialization
+    _preferred_endpoint = get_prefered_endpoint(
+        getattr(original_route.endpoint, "__module__"),
+        original_route.endpoint.__name__ + PREFERRED_SUFFIX,
+    )
+    # Need to adapt also the path regex to ensure it matches the path in different API versions
+    # Ensures the leading '^' is followed by any match and there's no '/' before final '$'
+    _new_regex_string = (
+        "^.*" + original_route.path_regex.pattern.lstrip("^").rstrip("/$") + "$"
+    )
+    _new_route: APIRoute = APIRoute(
+        path=original_route.path,
+        endpoint=(
+            _preferred_endpoint if _preferred_endpoint else original_route.endpoint
+        ),
+        response_model=original_route.response_model,
+        status_code=original_route.status_code,
+        tags=original_route.tags.copy() if original_route.tags else None,
+        dependencies=(
+            original_route.dependencies.copy() if original_route.dependencies else None
+        ),
+        summary=original_route.summary,
+        description=original_route.description,
+        response_description=original_route.response_description,
+        responses=original_route.responses.copy() if original_route.responses else None,
+        deprecated=original_route.deprecated,
+        name=original_route.name,
+        methods=original_route.methods.copy() if original_route.methods else None,
+        operation_id=original_route.operation_id,
+        response_model_include=original_route.response_model_include,
+        response_model_exclude=original_route.response_model_exclude,
+        response_model_by_alias=original_route.response_model_by_alias,
+        response_model_exclude_unset=original_route.response_model_exclude_unset,
+        response_model_exclude_defaults=original_route.response_model_exclude_defaults,
+        response_model_exclude_none=original_route.response_model_exclude_none,
+        include_in_schema=original_route.include_in_schema,
+        response_class=original_route.response_class,
+        dependency_overrides_provider=original_route.dependency_overrides_provider,
+        callbacks=original_route.callbacks.copy() if original_route.callbacks else None,
+        openapi_extra=(
+            original_route.openapi_extra.copy()
+            if original_route.openapi_extra
+            else None
+        ),
+        generate_unique_id_function=original_route.generate_unique_id_function,
+    )
+    _new_route.path_regex = re.compile(_new_regex_string)
+    return _new_route
+
+
+def get_prefered_endpoint(module_name: str, function_name: str) -> Optional[callable]:
+    """Get the function if it exists in the module, else return None."""
+    try:
+        _module = importlib.import_module(module_name)
+        _prefered_endpoint = getattr(_module, function_name)
+        return _prefered_endpoint if callable(_prefered_endpoint) else None
+    except Exception:
+        return None
+
+
+def group_routes_by_method(api_routes: List[APIRoute]) -> Dict[str, List[APIRoute]]:
+    """
+    Groups API routes by their HTTP method. This is important to ensure that the URLPath is matched
+    correctly, as different methods may share the same URLPath (e.g. GET, POST PUT, DELETE).
+    """
+    _method_to_routes = defaultdict(
+        list
+    )  # Automatically initializes new lists for new keys
+    for _route in api_routes:
+        if (
+            _route.methods
+        ):  # Ensure methods exist (should always be true for valid routes)
+            for method in _route.methods:
+                _method_to_routes[method].append(clone_and_adapt_route(_route))
+
+    return dict(_method_to_routes)
+
+
+class CleverSwarmAmqpAdapter(CleverThisServiceAdapter):
+    """
+    CleverSwarm AMQP Adapter for CleverSwarm API. Implements the interface to CleverSwarm API.
+    """
+
+    def __init__(
+        self, amq_configuration: AMQConfiguration, session: ClientSession = None
+    ):
+        super().__init__(amq_configuration, session)
+        self.routes = group_routes_by_method(api_routes=api.api_router.routes)
+        _amq_service: AMQService = AMQService(amq_configuration, self)
+        self.thread = Thread(target=_amq_service.run)
+        self.thread.start()
+
+    async def get_current_user(self, token: str) -> UserSchema:
+        """
+        Overrides the default get_current_user method to use the token from the AMQP message.
+        :param token: JWT token from the AMQP message
+        :return: UserSchema object
+        """
+        return await core.deps.get_current_user(token)
+
+    # ==================================================================================================
+    # =====================  C l e v e r S w a r m   A P I   m e t h o d s  ============================
+    # ==================================================================================================
+
+    async def get(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the GET requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the GET request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_no_body_payload(message, auth_user, "GET")
+
+    async def put(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the PUT requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the PUT request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_body_payload(message, auth_user, "PUT")
+
+    async def post(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the POST requests for the CleverSwarm API.
+        :param message: message from the AMQP that contains the POST request, including body and headers
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        return await self.handle_body_payload(message, auth_user, "POST")
+
+    async def head(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the HEAD requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "HEAD")
+
+    async def delete(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the DELETE requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "DELETE")
+
+    async def options(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the OPTIONS requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "OPTIONS")
+
+    async def patch(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
+        """
+        Handles the PATCH requests for the CleverSwarm API.
+        """
+        return await self.handle_no_body_payload(message, auth_user, "PATCH")
+
+    async def handle_no_body_payload(
+        self, message: AMQMessage, auth_user: UserSchema, method: str
+    ) -> AMQResponse:
+        """
+        Handles the requests that do not have a body payload.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        _routes = self.routes.get(method, [])
+        _path, _args = AMQDataParser.parse_get_url(message.path)
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                _args.update(_match.groupdict())
+                _args["authenticated_user"] = auth_user
+                # If the path matches, call the corresponding function
+                return await CleverThisServiceAdapter.call_cleverthis_api(
+                    message,
+                    _args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code,
+                    loop=self.loop,
+                )
+
+        return AMQResponseFactory.of_error_message(
+            message.id,
+            AMQErrorMessage(
+                404,
+                "Not Found",
+                f"Destination location [{message.path}] does not exists",
+            ),
+            message.trace_info,
+        )
+
+    async def handle_body_payload(
+        self, message: AMQMessage, auth_user: UserSchema, method: str
+    ) -> AMQResponse:
+        """
+        Handles the requests that have a body payload, including Multipart data.
+        :param message: message from the AMQP that contains the request
+        :param auth_user: user authenticated by the API Gateway, as UserSchem
+        :param method: HTTP method (POST, PUT, etc.)
+        :return: AMQResponse to be sent back to the API Gateway via AMQP
+        """
+        _form_data = AMQDataParser.parse_request_body(message)
+        _routes = self.routes.get(method, [])
+        _path, _query_args = AMQDataParser.parse_get_url(message.path)
+        for _route in _routes:
+            _match = _route.path_regex.match(_path)
+            if _match:
+                # If the path matches, call the corresponding function
+                _form_data.update(_match.groupdict())
+                _form_data.update(_query_args)
+                for _body_param in _route.dependant.body_params:
+                    if _body_param.name in _form_data:
+                        _file_data = _form_data[_body_param.name]
+                        _form_data[_body_param.name] = (
+                            _body_param.type_.__name__ == "UploadFile"
+                            and message.extra_data.get(_body_param.name)
+                            and UploadFile(
+                                file=pathlib.Path(
+                                    message.extra_data.get(_body_param.name)
+                                ).open("rb"),
+                                size=_file_data["size"],
+                                filename=_file_data["filename"],
+                                headers=Headers(
+                                    {"Content-Type": _file_data["mediaType"]}
+                                ),
+                            )
+                            or _form_data[_body_param.name]
+                        )
+
+                _verified_args = {}
+                _required_args = getattr(_route.endpoint, "__annotations__", {})
+                for arg in _required_args:
+                    if arg in _form_data:
+                        _verified_args[arg] = _form_data[arg]
+                    else:
+                        # needed for /login  input
+                        if arg == "authenticated_user":
+                            _verified_args["authenticated_user"] = auth_user
+                        if _required_args[arg].__name__ == "OAuth2PasswordRequestForm":
+                            try:
+                                _verified_args[arg] = OAuth2PasswordRequestForm(
+                                    **_form_data
+                                )
+                            except Exception as e:
+                                print(f"Error: {e}")
+
+                return await CleverThisServiceAdapter.call_cleverthis_api(
+                    message,
+                    _verified_args,
+                    api_method=_route.endpoint,
+                    success_code=_route.status_code if _route.status_code else 200,
+                    loop=self.loop,
+                )
+
+        return AMQResponseFactory.of_error_message(
+            message.id,
+            AMQErrorMessage(
+                404,
+                "Not Found",
+                f"Destination location [{message.path}] does not exists",
+            ),
+            message.trace_info,
+        )

cleverswarm/main.py

@@ -0,0 +1,8 @@
+from amqp.config.amq_configuration import AMQConfiguration
+
+from clever_swarm_adapter import CleverSwarmAmqpAdapter
+
+if __name__ == "__main__":
+    adapter = CleverSwarmAmqpAdapter(AMQConfiguration("./application.properties.local"))
+    print("Press ^C to exit...")
+    adapter.thread.join()

pyproject.toml

@@ -4,17 +4,43 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "amq_adapter"
-version = "0.2.6"
-description = "The Python implementation of the AMQ Adaptor for CleverMicro"
+version = "0.2.30"
+description = "The Python implementation of the AMQ Adapter for CleverMicro"
 readme = "README.md"
+authors = [
+    { name = "Stanislav Hejny", email = "stanislav.hejny@cleverthis.com" }
+]
 license = { text = "Apache License 2.0" }
+urls = { Homepage = "https://git.cleverthis.com/clevermicro/amq-adapter-python" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.11"
 dependencies = [
-    "aiohttp~=3.11.11",
-    "pika~=1.3.2",
-    "fastapi==0.115.6",
+    "aio-pika",
+    "aiohttp",
+    "fastapi",
     "opentelemetry-api",
     "opentelemetry-sdk",
     "opentelemetry-exporter-otlp",
     "opentelemetry-exporter-prometheus",
-    "opentelemetry-instrumentation-pika"
+    "opentelemetry-instrumentation-pika",
+    "pika",
+    "psutil",
+    "python_multipart"
 ]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["amqp*"]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "build",
+    "twine"
+]
+

requirements.txt

@@ -1,8 +0,0 @@
-aiohttp~=3.11.11
-fastapi==0.115.6
-aio_pika~=9.5.5
-opentelemetry-api
-opentelemetry-sdk
-opentelemetry-exporter-otlp
-opentelemetry-exporter-prometheus
-opentelemetry-instrumentation-pika

setup.py

@@ -1,30 +0,0 @@
-from setuptools import setup, find_packages
-
-setup(
-    name="amqp",
-    version="0.2.6",
-    author="Stanislav Hejny",
-    author_email="stanislav.hejny@cleverthis.com",
-    description="Adapter for RabbitMQ to integrate a CleverThis python-code Service with CleverMicro platform",
-    long_description=open("README.md").read(),
-    long_description_content_type="text/markdown",
-    url="https://git.cleverthis.com/cleverthis/clevermicro/amq-adapter-python",
-    packages=find_packages(),
-    install_requires=[
-        # dependencies here
-        "aiohttp~=3.11.11",
-        "fastapi==0.115.6",
-        "pika~=1.3.2",
-        "opentelemetry-api",
-        "opentelemetry-sdk",
-        "opentelemetry-exporter-otlp",
-        "opentelemetry-exporter-prometheus",
-        "opentelemetry-instrumentation-pika"
-    ],
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],
-    python_requires=">=3.11",
-)

tests/adapter/test_amq_route_factory.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+
+from amqp.adapter.amq_route_factory import AMQRouteFactory, NULL_ROUTE
+from amqp.model.model import AMQRoute
+
+
+class TestAMQRouteFactory(TestCase):
+    def test_from_string(self):
+        _f = AMQRouteFactory()
+        _r: AMQRoute = _f.from_string(
+            "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+        )
+        self.assertEqual("", _r.exchange)
+        self.assertEqual("amq-clever-pybook", _r.queue)
+        self.assertEqual(5, _r.timeout)
+        _r2: AMQRoute = _f.from_string(
+            "amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#:5"
+        )
+        self.assertEqual(NULL_ROUTE, _r2)
+
+    def test_validate(self):
+        _f = AMQRouteFactory()
+        self.assertTrue(
+            _f.validate(
+                "amq-clever-pybook::amq-clever-pybook:clever.pybook.localhost.#:5:0"
+            )
+        )
+        self.assertFalse(
+            _f.validate("amq-clever-pybook:amq-clever-pybook:clever.pybook.localhost.#")
+        )

tests/adapter/test_cleverthis_service_adapter.py

@@ -0,0 +1,227 @@
+import unittest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from aiohttp import ClientResponse
+
+from amqp.adapter.cleverthis_service_adapter import AMQMessage, CleverThisServiceAdapter
+from amqp.config.amq_configuration import AMQConfiguration
+from amqp.model.model import DataMessage, DataResponse
+
+
+class TestCleverThisServiceAdapter(unittest.TestCase):
+    def setUp(self):
+        # Mock AMQConfiguration
+        self.mock_amq_config = AMQConfiguration("./application.properties.local")
+
+        # Mock ClientSession
+        self.mock_session = AsyncMock()
+
+        # Create adapter instance
+        self.adapter = CleverThisServiceAdapter(
+            amq_configuration=self.mock_amq_config, session=self.mock_session
+        )
+
+    def test_init(self):
+        self.assertEqual(self.adapter.service_host, "localhost")
+        self.assertEqual(self.adapter.service_port, 8080)
+        self.assertFalse(self.adapter.require_authenticated_user)
+
+    def test_get_content_type(self):
+        test_cases = [
+            ({"Content-Type": ["application/json"]}, "application/json"),
+            ({"content-type": ["text/xml"]}, "text/xml"),
+            ({"CONTENT-TYPE": ["text/plain; charset=utf-8"]}, "text/plain"),
+            ({}, "application/json"),
+            ({"Other-Header": ["value"]}, "application/json"),
+        ]
+
+        for headers, expected in test_cases:
+            with self.subTest(headers=headers):
+                result = self.adapter.get_content_type(headers)
+                self.assertEqual(result, expected)
+
+    async def test_get_current_user(self):
+        # Base implementation should return None
+        result = await self.adapter.get_current_user("test_token")
+        self.assertIsNone(result)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_unauthenticated(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        # Test when authentication is required but not provided
+        response = await self.adapter.on_message(mock_message)
+        self.assertEqual(response.response_code, 401)
+        mock_get_user.assert_not_called()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    @patch.object(CleverThisServiceAdapter, "get")
+    async def test_on_message_authenticated(self, mock_get, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+        mock_message.path = "/some/path"
+        mock_message.method = "GET"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+        mock_get.return_value = mock_response
+
+        response = await self.adapter.on_message(mock_message)
+
+        self.assertEqual(response, mock_response)
+        mock_get_user.assert_called_once_with("valid_token")
+        mock_get.assert_called_once_with(mock_message, mock_user)
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_login_path(self, mock_get_user):
+        mock_message = MagicMock(spec=AMQMessage)
+        mock_message.headers = {}
+        mock_message.path = "/login"
+        mock_message.method = "POST"
+        mock_message.id = "123"
+        mock_message.trace_info = {}
+
+        mock_response = MagicMock(spec=DataResponse)
+        with patch.object(
+            self.adapter, "post", return_value=mock_response
+        ) as mock_post:
+            response = await self.adapter.on_message(mock_message)
+            self.assertEqual(response, mock_response)
+            mock_get_user.assert_not_called()
+            mock_post.assert_called_once()
+
+    @patch.object(CleverThisServiceAdapter, "get_current_user")
+    async def test_on_message_all_methods(self, mock_get_user):
+        mock_user = MagicMock()
+        mock_get_user.return_value = mock_user
+
+        mock_response = MagicMock(spec=DataResponse)
+
+        methods = ["GET", "PUT", "POST", "DELETE", "HEAD"]
+        for method in methods:
+            with self.subTest(method=method):
+                mock_message = MagicMock(spec=AMQMessage)
+                mock_message.headers = {"Authorization": ["Bearer valid_token"]}
+                mock_message.path = "/some/path"
+                mock_message.method = method
+                mock_message.id = "123"
+                mock_message.trace_info = {}
+
+                with patch.object(
+                    self.adapter, method.lower(), return_value=mock_response
+                ) as mock_method:
+                    response = await self.adapter.on_message(mock_message)
+                    self.assertEqual(response, mock_response)
+                    mock_method.assert_called_once()
+
+    async def test_get_with_session(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {"X-Test": "value"}
+        mock_message.trace_info = {"trace-id": "123"}
+        mock_message.id = "msg123"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.get.return_value = mock_response
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 200)
+        self.mock_session.get.assert_called_once_with(
+            "http://testhost:8080/api/test",
+            headers={"X-Test": "value", "trace-id": "123"},
+        )
+
+    async def test_get_without_session(self):
+        self.adapter.session = None
+
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.id = "msg123"
+        mock_message.trace_info = {}
+
+        response = await self.adapter.get(mock_message, None)
+
+        self.assertEqual(response.response_code, 404)
+
+    async def test_put_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 200
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "success"}')
+
+        self.mock_session.put.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "_authorize_forward_request",
+            wraps=self.adapter._authorize_forward_request,
+        ) as mock_handler:
+            response = await self.adapter.put(mock_message, None)
+
+            self.assertEqual(response.response_code, 200)
+            mock_handler.assert_called_once()
+
+    async def test_post_with_form_handling(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.path = "/api/test"
+        mock_message.headers = {}
+        mock_message.trace_info = {}
+        mock_message.id = "msg123"
+        mock_message.body = b"test=value"
+
+        mock_response = MagicMock(spec=ClientResponse)
+        mock_response.status = 201
+        mock_response.content_type = "application/json"
+        mock_response.read = AsyncMock(return_value=b'{"result": "created"}')
+
+        self.mock_session.post.return_value = mock_response
+
+        with patch.object(
+            self.adapter,
+            "_authorize_forward_request",
+            wraps=self.adapter._authorize_forward_request,
+        ) as mock_handler:
+            response = await self.adapter.post(mock_message, None)
+
+            self.assertEqual(response.response_code, 201)
+            mock_handler.assert_called_once()
+
+    async def test_head_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.head(mock_message, None)
+
+    async def test_delete_not_implemented(self):
+        mock_message = MagicMock(spec=DataMessage)
+        mock_message.id = "msg123"
+
+        # Assuming delete is not implemented in base class
+        with self.assertRaises(NotImplementedError):
+            await self.adapter.delete(mock_message, None)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/adapter/test_data_message_factory.py

@@ -0,0 +1,129 @@
+from datetime import datetime, timezone
+from unittest import TestCase
+
+from amqp.adapter.data_message_factory import DataMessageFactory
+
+
+class DataMessageFactoryTest(TestCase):
+    def test_from_bytes(self):
+        raw = "\002\027A64f3b5af4d00000a4000|m=POST|d=clever-pybook.localhost|p=/debug|h={Accept=[*/*],X-Forwarded-Host=[clever-pybook.localhost:8085],User-Agent=[curl/8.7.1],X-Forwarded-Proto=[http],X-Forwarded-For=[172.18.0.1],Host=[clever-pybook.localhost:8085],Accept-Encoding=[gzip],Content-Length=[1475],X-Forwarded-Port=[8085],X-Real-Ip=[172.18.0.1],X-Forwarded-Server=[c5945bef67d1],Content-Type=[multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v]}|t={traceparent=00-bed6eb7de0d8ccf4c80b145a4bfec357-8d1f3bafc471bf0f-01}|b=TEST_ME"
+        msg = DataMessageFactory.from_bytes(raw.encode("utf-8"))
+        self.assertIsNotNone(msg.id)
+        self.assertEqual(
+            msg.id.as_string(),
+            "64F3B5AF4D00000A4000".lower(),
+            "SnowflakeID incorrectly parsed.",
+        )
+        self.assertEqual(
+            msg.headers["Content-Type"][0],
+            "multipart/form-data; boundary=------------------------j1RjIqLrFH07XMsyMvpS1v",
+        )
+
+    def test_generate_snowflake_id(self):
+        _id = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        self.assertEqual(
+            "00001000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _id = DataMessageFactory(2).generate_snowflake_id()
+        self.assertEqual(
+            "00002000", _id.as_string()[-8:], "Instance ID not set correctly"
+        )
+
+        _f = DataMessageFactory(127)
+        _i0 = _f.generate_snowflake_id()
+        _i1 = _f.generate_snowflake_id()
+        _i2 = _f.generate_snowflake_id()
+        _i3 = _f.generate_snowflake_id()
+        self.assertEqual(
+            "0007F000".lower(),
+            _i0.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F001".lower(),
+            _i1.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F002".lower(),
+            _i2.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+        self.assertEqual(
+            "0007F003".lower(),
+            _i3.as_string()[-8:],
+            "Instance ID and sequence not set correctly",
+        )
+
+    def test_create_request_message(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        self.assertEqual("localhost", _req.domain, "Domain not set correctly")
+        self.assertEqual("/test/me", _req.path, "Context path not set correctly")
+        self.assertEqual("v1", _req.headers["k1"][0], "Headers not set correctly")
+        self.assertEqual(b"Quick brown fox", _req.body(), "Body not correctly decoded")
+
+    def test_serialize(self):
+        _f = DataMessageFactory(34)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _ser = _f.serialize(_req)
+        self.assertEqual(65, _ser[2])
+        _deser = _f.from_bytes(_ser)
+        self.assertEqual(_req.id.as_string(), _deser.id.as_string())
+        self.assertEqual("localhost", _deser.domain)
+        self.assertEqual("/test/me", _deser.path)
+        self.assertEqual(b"Quick brown fox", _deser.body())
+        self.assertEqual("v2", _deser.headers["k2"][0])
+
+    def test_amq_message_routing_key(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _key = _f.amq_message_routing_key(_req)
+        self.assertEqual(
+            "localhost.test.me",
+            _key,
+            "Message routing key wrongly extracted from the message",
+        )
+
+    def test_get_timestamp_from_id(self):
+        _f = DataMessageFactory(35)
+        _timestamp = _f.get_timestamp_from_id(_f.generate_snowflake_id())
+        _x_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
+        self.assertLessEqual(
+            _x_timestamp - _timestamp,
+            1,
+            "Timestamp in SnowflakeId differs too much from current time",
+        )
+
+    def test_to_string(self):
+        _f = DataMessageFactory(35)
+        _req = _f.create_request_message(
+            "PUT",
+            "localhost",
+            "/test/me",
+            {"k1": ["v1"], "k2": ["v2"]},
+            "Quick brown fox",
+        )
+        _as_str = _f.to_string(_req)
+        self.assertGreater(
+            _as_str.index("domain='localhost'"), 0, "Expected content not found"
+        )

tests/adapter/test_data_response_factory.py

@@ -1,36 +1,40 @@
 from unittest import TestCase
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
-from amqp.adapter.amq_response_factory import AMQResponseFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
+from amqp.adapter.data_response_factory import DataResponseFactory
 from amqp.model.snowflake_id import SnowflakeId
 
 
-class TestAMQResponseFactory(TestCase):
+class TestDataResponseFactory(TestCase):
 
     def test_create_async_response_message(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _resp = _f.create_async_response_message(_g.generate_snowflake_id())
         self.assertEqual(200, _resp.response_code)
         self.assertEqual(b"OK", _resp.response)
 
     def test_serialize(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
         _resp = _f.create_async_response_message(_snowflakeId)
         _ser = _f.serialize(_resp)
-        self.assertEqual(2, str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST+_snowflakeId.as_string()), "Magic byte and SnowflakeId not found")
+        self.assertEqual(
+            2,
+            str(_ser[2:]).index(_f.MAGIC_BYTE_HTTP_REQUEST + _snowflakeId.as_string()),
+            "Magic byte and SnowflakeId not found",
+        )
         self.assertGreater(str(_ser[2:]).index("|e=|"), 2)
-        self.assertEqual((_ser[0]+_ser[1]), len(_ser) - 2 - len('T0s='))
+        self.assertEqual((_ser[0] + _ser[1]), len(_ser) - 2 - len("T0s="))
 
     def test_from_bytes(self):
-        _f = AMQResponseFactory()
-        _g = AMQMessageFactory(111)
+        _f = DataResponseFactory()
+        _g = DataMessageFactory(111)
         _snowflakeId: SnowflakeId = _g.generate_snowflake_id()
         _resp = _f.create_async_response_message(_snowflakeId)
         _ser = _f.serialize(_resp)
         _deser = _f.from_bytes(_ser)
         self.assertEqual(_snowflakeId.as_string(), _deser.id)
         self.assertEqual(200, _deser.response_code)
-        self.assertEqual(b'OK', _deser.response)
+        self.assertEqual(b"OK", _deser.response)

tests/adapter/test_file_handler.py

@@ -0,0 +1,50 @@
+import os
+import tempfile
+from unittest import TestCase
+
+from amqp.adapter.file_handler import StreamingFileHandler
+
+
+class TestFileHandler(TestCase):
+    def test_ensure_directory_exists(self):
+        """
+        Test function for ensure_directory_exists.  Creates a temporary directory
+        and files to test the versioning logic.
+        """
+
+        # Create a temporary directory
+        file_handler = StreamingFileHandler()
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Test case 1: File does not exist
+            filepath1 = os.path.join(temp_dir, "test1.txt")
+            result1 = file_handler.ensure_directory_exists(filepath1)
+            assert result1 == filepath1, f"Test Case 1 Failed: {result1} != {filepath1}"
+            # Create the file
+            with open(filepath1, "w") as f:
+                f.write("test")
+
+            # Test case 2: File exists, should create versioned file
+            filepath2 = os.path.join(temp_dir, "test2.txt")
+            result2 = file_handler.ensure_directory_exists(filepath2)
+            assert result2 == filepath2, f"Test Case 2 Failed: {result2} != {filepath2}"
+            with open(filepath2, "w") as f:
+                f.write("test")
+            result2_v1 = file_handler.ensure_directory_exists(filepath2)
+            assert result2_v1 == os.path.join(
+                temp_dir, "test2.1.txt"
+            ), f"Test Case 2 Version 1 Failed: {result2_v1} != {os.path.join(temp_dir, 'test2.1.txt')}"
+
+            # Create a few versioned files
+            with open(os.path.join(temp_dir, "test3.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.1.txt"), "w") as f:
+                f.write("test")
+            with open(os.path.join(temp_dir, "test3.2.txt"), "w") as f:
+                f.write("test")
+            filepath3 = os.path.join(temp_dir, "test3.txt")
+            result3_v3 = file_handler.ensure_directory_exists(filepath3)
+            assert result3_v3 == os.path.join(
+                temp_dir, "test3.3.txt"
+            ), f"Test Case 3 Version 3 Failed: {result3_v3} != {os.path.join(temp_dir, 'test3.3.txt')}"
+
+            print("All test cases passed!")

tests/adapter/test_pydantic_serializer.py

@@ -0,0 +1,38 @@
+from unittest import TestCase
+
+from amqp.adapter.pydantic_serializer import parse_url_query
+
+
+class PydanticSerializerTest(TestCase):
+    def test_serialize_object(self):
+        assert True
+
+    def test_deserialize_object(self):
+        assert True
+
+    def test__convert_value(self):
+        assert True
+
+    def test__is_uuid_string(self):
+        assert True
+
+    def test__is_datetime_string(self):
+        assert True
+
+    def test__is_int_string(self):
+        assert True
+
+    def test_serialize_to_json(self):
+        assert True
+
+    def test_parse_url_query(self):
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741?response_type=JsonFile"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {"response_type": "JsonFile"}
+        path, args = parse_url_query(
+            "http://localhost:8080/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        )
+        assert path == "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741"
+        assert args == {}

tests/adapter/test_service_message_factory.py

@@ -1,28 +1,36 @@
 from unittest import TestCase
 
-from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory, RS, INVALID_MESSAGE
-from amqp.model.model import CleverMicroServiceMessage
+from amqp.adapter.service_message_factory import (
+    ServiceMessageFactory,
+    RS,
+    INVALID_MESSAGE,
+)
+from amqp.model.model import ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
 
 
-class CleverMicroServiceMessageTest(TestCase):
+class ServiceMessageTest(TestCase):
 
     def test_to_string(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory('bumble-bee')
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
+        _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee")
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA, "Duck", "Donald"
+        )
         message_str = _f.to_string(message)
         self.assertTrue("|type=01|" in message_str)
         self.assertTrue("|body=Duck" in message_str)
-        message: CleverMicroServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald")
+        message: ServiceMessage = _f.of(
+            ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald"
+        )
         message_str = _f.to_string(message)
         self.assertTrue("|type=03|" in message_str)
         self.assertTrue("|body=Duck" in message_str)
         self.assertTrue("|replyTo=bumble-bee" in message_str)
-        nullMsg  = _f.to_string(None)
+        nullMsg = _f.to_string(None)
         self.assertEqual("null", nullMsg)
 
     def test_from(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         deserialized = _f.from_bytes(serialized)
@@ -31,11 +39,11 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.recipient_name == "Donald"
         assert deserialized.message == "Duck"
         assert deserialized.reply_to == "amqp-router"
-        invalidMsg = _f.from_bytes(b'wrong')
+        invalidMsg = _f.from_bytes(b"wrong")
         self.assertEqual(invalidMsg, INVALID_MESSAGE)
 
     def test_base64(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.as_base64(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert b"|81|" in serialized
@@ -43,9 +51,9 @@ class CleverMicroServiceMessageTest(TestCase):
         assert deserialized.is_valid()
         assert deserialized.message_type == ServiceMessageType.ROUTING_DATA
         assert deserialized.message == "Duck"
-    
+
     def test_serialize(self):
-        _f: CleverMicroServiceMessageFactory = CleverMicroServiceMessageFactory("amqp-router")
+        _f: ServiceMessageFactory = ServiceMessageFactory("amqp-router")
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value)

tests/model/test_snowflake_id.py

@@ -1,48 +1,48 @@
 from unittest import TestCase
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.model.snowflake_id import SnowflakeId
 
 
 class SnowflakeIdTest(TestCase):
 
     def test_equals(self):
-        _id_1 = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        _id_2 = AMQMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
         self.assertFalse(_id_1 == _id_2)
         _id_3 = SnowflakeId(_id_1.get_bytes())
         self.assertTrue(_id_1 == _id_3)
 
     def test_lt(self):
-        _id_1 = AMQMessageFactory.get_instance(1).generate_snowflake_id()
-        _id_2 = AMQMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_1 = DataMessageFactory.get_instance(1).generate_snowflake_id()
+        _id_2 = DataMessageFactory.get_instance(1).generate_snowflake_id()
         self.assertTrue(_id_1 < _id_2)
 
-
     def test_from_hex(self):
-        _factory = AMQMessageFactory.get_instance(1)
+        _factory = DataMessageFactory.get_instance(1)
         _id_1 = _factory.generate_snowflake_id()
         _id_1_ser = _id_1.as_string()
         _id_2 = SnowflakeId.from_hex(_id_1_ser)
         self.assertTrue(_id_1 == _id_2)
         with self.assertRaises(RuntimeError) as context:
             SnowflakeId.from_hex("100")
-        self.assertTrue(f"MessageId is {SnowflakeId.SNOWFLAKE_ID_WIDTH * 8} bits wide, it is represented by HEX string "
-                        in str(context.exception))
+        self.assertTrue(
+            f"MessageId is {SnowflakeId.SNOWFLAKE_ID_WIDTH * 8} bits wide, it is represented by HEX string "
+            in str(context.exception)
+        )
 
     def test_get_bytes(self):
-        _factory = AMQMessageFactory.get_instance(1)
+        _factory = DataMessageFactory.get_instance(1)
         _id_1 = _factory.generate_snowflake_id()
         _id_1_ser = _id_1.get_bytes()
         self.assertEqual(SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1_ser))
 
     def test_get_bits(self):
-        _factory = AMQMessageFactory.get_instance(1)
+        _factory = DataMessageFactory.get_instance(1)
         _id_1 = _factory.generate_snowflake_id()
         self.assertEqual(2, len(_id_1.get_bits()))
 
-
     def test_as_string(self):
-        _factory = AMQMessageFactory.get_instance(1)
+        _factory = DataMessageFactory.get_instance(1)
         _id_1 = _factory.generate_snowflake_id()
         self.assertEqual(2 * SnowflakeId.SNOWFLAKE_ID_WIDTH, len(_id_1.as_string()))

tests/router/test_route_database.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 import pytest
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.adapter.amq_route_factory import AMQRouteFactory
 from amqp.model.model import AMQRoute
 from amqp.router.route_database import RouteDatabase
@@ -28,7 +28,9 @@ class TestRouteDatabase(TestCase):
 
     def test_get_routes(self):
         _amq_factory = AMQRouteFactory()
-        _route: AMQRoute = _amq_factory.from_string("amq-clever-pybook::clever-pybook:clever.pybook.localhost.#:5:0")
+        _route: AMQRoute = _amq_factory.from_string(
+            "amq-clever-pybook::clever-pybook:clever.pybook.localhost.#:5:0"
+        )
         _routes = self._database.get_routes()
         self.assertIsNotNone(_routes)
         self.assertEqual(3, len(_routes))
@@ -75,9 +77,8 @@ class TestRouteDatabase(TestCase):
         self.assertTrue(ROUTE_2 in wrap)
 
     def test_find_route(self):
-        message = AMQMessageFactory.get_instance(111).create_request_message(
-              "POST", "clever3-book.localhost",
-              "/aa/bb?cc=d", {}, ""
+        message = DataMessageFactory.get_instance(111).create_request_message(
+            "POST", "clever3-book.localhost", "/aa/bb?cc=d", {}, ""
         )
         route = self._database.find_route(message.domain, message.path)
         self.assertIsNotNone(route)

tests/router/test_router_base.py

@@ -3,7 +3,7 @@ from unittest import TestCase
 
 import pytest
 
-from amqp.adapter.amq_message_factory import AMQMessageFactory
+from amqp.adapter.data_message_factory import DataMessageFactory
 from amqp.adapter.amq_route_factory import AMQRouteFactory
 from amqp.model.model import AMQRoute
 from amqp.router.router_base import RouterBase
@@ -23,7 +23,7 @@ class TestRouterBase(TestCase):
     @pytest.fixture(autouse=True)
     def setup_each_test(self):
         self.base = RouterBase()
-        self.factory = AMQMessageFactory.get_instance(111)
+        self.factory = DataMessageFactory.get_instance(111)
         if len(self.base.get_routes()) == 0:
             r1 = AMQRouteFactory.from_string(ROUTE_1)
             r2 = AMQRouteFactory.from_string(ROUTE_2)
@@ -44,7 +44,9 @@ class TestRouterBase(TestCase):
         self.assertEqual("aa.bb.cc", sanitize_as_rabbitmq_name("aa///bb////cc"))
 
     def test_get_routing_key(self):
-        message = self.factory.create_request_message("POST", "test.me.localhost", "/aa/bb?cc=d", {}, "")
+        message = self.factory.create_request_message(
+            "POST", "test.me.localhost", "/aa/bb?cc=d", {}, ""
+        )
         rk = self.base.get_routing_key(message)
         self.assertEqual("test.me.localhost.aa.bb", rk)
 
@@ -79,9 +81,8 @@ class TestRouterBase(TestCase):
         self.assertIsNone(_route)
 
     def test_find_route_by_message(self):
-        message = AMQMessageFactory.get_instance(111).create_request_message(
-              "POST", "clever3-book.localhost",
-              "/aa/bb?cc=d", {}, ""
+        message = DataMessageFactory.get_instance(111).create_request_message(
+            "POST", "clever3-book.localhost", "/aa/bb?cc=d", {}, ""
         )
         _route: AMQRoute = self.base.find_route_by_message(message)
         self.assertIsNotNone(_route)
@@ -91,7 +92,7 @@ class TestRouterBase(TestCase):
         init_routes: Set[AMQRoute] = self.base.get_routes().copy()
         self.base.add_routes(ROUTE_6 + ",amq-dlq:DLX:DeadLetterQueue:#:0:0,:::::,:::")
         modified = self.base.get_routes()
-        self.assertEqual(len(init_routes)+1, len(modified))
+        self.assertEqual(len(init_routes) + 1, len(modified))
         wr = self.base.wrap_routes()
         self.assertFalse("DLX" in wr)
 

tools/create_exchange.py

@@ -0,0 +1,90 @@
+import asyncio
+import aio_pika
+import os
+import argparse
+import sys
+
+from aio_pika.abc import AbstractRobustExchange
+
+
+async def create_exchange(loop, rabbit_user, rabbit_password, rabbit_host):
+    """
+    Asynchronously connects to RabbitMQ, declares a direct exchange,
+    and prints the result.  Handles potential connection errors.
+    """
+    exchange_name = "cleverthis.clevermicro.management"
+    connection = None  # Keep track of the connection
+
+    try:
+        # Construct the connection URI from environment variables
+        connection_uri = f"amqp://{rabbit_user}:{rabbit_password}@{rabbit_host}/"
+
+        # Attempt to establish the connection
+        connection = await aio_pika.connect_robust(connection_uri, loop=loop)
+        print(f"Connected to RabbitMQ: {connection_uri}")
+
+        # Create a channel
+        channel = await connection.channel()
+
+        # Declare the exchange
+        exchange: AbstractRobustExchange = await channel.declare_exchange(
+            exchange_name, aio_pika.ExchangeType.DIRECT
+        )
+        print(f"Exchange '{exchange.name}' created successfully.")
+
+    except aio_pika.exceptions.AMQPConnectionError as e:
+        print(f"Error: Could not connect to RabbitMQ.  Details: {e}")
+        sys.exit(1)  # Exit with an error code
+    except aio_pika.exceptions.AMQPChannelError as e:
+        print(f"Error creating channel or exchange: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+        sys.exit(1)
+    finally:
+        # Ensure the connection is closed, even if errors occur
+        if connection:
+            await connection.close()
+            print("Connection to RabbitMQ closed.")
+
+def main():
+    """
+    Main function to parse command-line arguments, retrieve RabbitMQ
+    credentials from environment variables, and run the
+    async exchange creation.
+    """
+    # Set up argument parsing
+    parser = argparse.ArgumentParser(description="Create a RabbitMQ exchange.")
+    parser.add_argument(
+        "--host",
+        default="127.0.0.1",
+        help="RabbitMQ host (default: localhost)",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+    rabbit_host = args.host
+
+    # Get RabbitMQ credentials from environment variables
+    rabbit_user = os.environ.get("RABBIT_MQ_USER")
+    rabbit_password = os.environ.get("RABBIT_MQ_PASSWORD")
+
+    if not rabbit_user or not rabbit_password:
+        print(
+            "Error: RabbitMQ credentials not found in environment variables.\n"
+            "Please set RABBIT_MQ_USER and RABBIT_MQ_PASSWORD."
+        )
+        sys.exit(1)  # Exit with an error code
+
+    # Create the event loop
+    loop = asyncio.get_event_loop()
+
+    # Run the asynchronous exchange creation
+    loop.run_until_complete(create_exchange(loop, rabbit_user, rabbit_password, rabbit_host))
+
+    # Close the loop
+    loop.close()
+
+
+if __name__ == "__main__":
+    main()

tools/delete_queues.py

@@ -0,0 +1,48 @@
+import requests
+
+
+def delete_queues_with_prefix_http(
+    host="localhost",
+    port=15672,
+    username="springuser",
+    password="TheCleverWho",
+    prefix="cm-",
+):
+    """
+    CLI utility. Delete queues using RabbitMQ HTTP API.
+    """
+    base_url = f"http://{host}:{port}/api/queues/%2F"
+
+    try:
+        response = requests.get(base_url, auth=(username, password))
+        response.raise_for_status()
+
+        queues = [q["name"] for q in response.json() if q["name"].startswith(prefix)]
+
+        if not queues:
+            print(f"No queues found with prefix '{prefix}'")
+            return
+
+        print(f"Found {len(queues)} queues with prefix '{prefix}':")
+        for queue in queues:
+            print(f" - {queue}")
+
+        for queue in queues:
+            try:
+                delete_url = f"{base_url}/{queue}"
+                response = requests.delete(delete_url, auth=(username, password))
+                response.raise_for_status()
+                print(f"Successfully deleted queue: {queue}")
+            except Exception as e:
+                print(f"Failed to delete queue {queue}: {str(e)}")
+
+        print("Queue deletion process completed.")
+
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+
+
+if __name__ == "__main__":
+    delete_queues_with_prefix_http()
+    delete_queues_with_prefix_http(prefix="amq.gen--")
+    delete_queues_with_prefix_http(prefix="amq_")